Skip to content

Commit 85da47f

Browse files
avivhalfonavivhalfon
authored
fix(langchain): added vendors to llm calls (#3165)
1 parent 6704f15 commit 85da47f

File tree

5 files changed

+175
-14
lines changed

5 files changed

+175
-14
lines changed

‎packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/callback_handler.py

Lines changed: 34 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,9 @@
4141
set_llm_request,
4242
set_request_params,
4343
)
44+
from opentelemetry.instrumentation.langchain.vendor_detection import (
45+
detect_vendor_from_class,
46+
)
4447
from opentelemetry.instrumentation.langchain.utils import (
4548
CallbackFilteredJSONEncoder,
4649
dont_throw,
@@ -63,6 +66,25 @@
6366
from opentelemetry.trace.status import Status, StatusCode
6467

6568

69+
def _extract_class_name_from_serialized(serialized: Optional[dict[str, Any]]) -> str:
70+
"""
71+
Extract class name from serialized model information.
72+
73+
Args:
74+
serialized: Serialized model information from LangChain callback
75+
76+
Returns:
77+
Class name string, or empty string if not found
78+
"""
79+
class_id = (serialized or {}).get("id", [])
80+
if isinstance(class_id, list) and len(class_id) > 0:
81+
return class_id[-1]
82+
elif class_id:
83+
return str(class_id)
84+
else:
85+
return ""
86+
87+
6688
def _message_type_to_role(message_type: str) -> str:
6789
if message_type == "human":
6890
return "user"
@@ -258,6 +280,7 @@ def _create_llm_span(
258280
name: str,
259281
request_type: LLMRequestTypeValues,
260282
metadata: Optional[dict[str, Any]] = None,
283+
serialized: Optional[dict[str, Any]] = None,
261284
) -> Span:
262285
workflow_name = self.get_workflow_name(parent_run_id)
263286
entity_path = self.get_entity_path(parent_run_id)
@@ -271,7 +294,10 @@ def _create_llm_span(
271294
entity_path=entity_path,
272295
metadata=metadata,
273296
)
274-
_set_span_attribute(span, SpanAttributes.LLM_SYSTEM, "Langchain")
297+
298+
vendor = detect_vendor_from_class(_extract_class_name_from_serialized(serialized))
299+
300+
_set_span_attribute(span, SpanAttributes.LLM_SYSTEM, vendor)
275301
_set_span_attribute(span, SpanAttributes.LLM_REQUEST_TYPE, request_type.value)
276302

277303
return span
@@ -384,7 +410,7 @@ def on_chat_model_start(
384410

385411
name = self._get_name_from_callback(serialized, kwargs=kwargs)
386412
span = self._create_llm_span(
387-
run_id, parent_run_id, name, LLMRequestTypeValues.CHAT, metadata=metadata
413+
run_id, parent_run_id, name, LLMRequestTypeValues.CHAT, metadata=metadata, serialized=serialized
388414
)
389415
set_request_params(span, kwargs, self.spans[run_id])
390416
if should_emit_events():
@@ -410,7 +436,7 @@ def on_llm_start(
410436

411437
name = self._get_name_from_callback(serialized, kwargs=kwargs)
412438
span = self._create_llm_span(
413-
run_id, parent_run_id, name, LLMRequestTypeValues.COMPLETION
439+
run_id, parent_run_id, name, LLMRequestTypeValues.COMPLETION, serialized=serialized
414440
)
415441
set_request_params(span, kwargs, self.spans[run_id])
416442
if should_emit_events():
@@ -478,11 +504,12 @@ def on_llm_end(
478504
)
479505

480506
# Record token usage metrics
507+
vendor = span.attributes.get(SpanAttributes.LLM_SYSTEM, "Langchain")
481508
if prompt_tokens > 0:
482509
self.token_histogram.record(
483510
prompt_tokens,
484511
attributes={
485-
SpanAttributes.LLM_SYSTEM: "Langchain",
512+
SpanAttributes.LLM_SYSTEM: vendor,
486513
SpanAttributes.LLM_TOKEN_TYPE: "input",
487514
SpanAttributes.LLM_RESPONSE_MODEL: model_name or "unknown",
488515
},
@@ -492,7 +519,7 @@ def on_llm_end(
492519
self.token_histogram.record(
493520
completion_tokens,
494521
attributes={
495-
SpanAttributes.LLM_SYSTEM: "Langchain",
522+
SpanAttributes.LLM_SYSTEM: vendor,
496523
SpanAttributes.LLM_TOKEN_TYPE: "output",
497524
SpanAttributes.LLM_RESPONSE_MODEL: model_name or "unknown",
498525
},
@@ -506,10 +533,11 @@ def on_llm_end(
506533

507534
# Record duration
508535
duration = time.time() - self.spans[run_id].start_time
536+
vendor = span.attributes.get(SpanAttributes.LLM_SYSTEM, "Langchain")
509537
self.duration_histogram.record(
510538
duration,
511539
attributes={
512-
SpanAttributes.LLM_SYSTEM: "Langchain",
540+
SpanAttributes.LLM_SYSTEM: vendor,
513541
SpanAttributes.LLM_RESPONSE_MODEL: model_name or "unknown",
514542
},
515543
)

‎packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/span_utils.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -336,11 +336,13 @@ def set_chat_response_usage(
336336
cache_read_tokens,
337337
)
338338
if record_token_usage:
339+
vendor = span.attributes.get(SpanAttributes.LLM_SYSTEM, "Langchain")
340+
339341
if input_tokens > 0:
340342
token_histogram.record(
341343
input_tokens,
342344
attributes={
343-
SpanAttributes.LLM_SYSTEM: "Langchain",
345+
SpanAttributes.LLM_SYSTEM: vendor,
344346
SpanAttributes.LLM_TOKEN_TYPE: "input",
345347
SpanAttributes.LLM_RESPONSE_MODEL: model_name,
346348
},
@@ -350,7 +352,7 @@ def set_chat_response_usage(
350352
token_histogram.record(
351353
output_tokens,
352354
attributes={
353-
SpanAttributes.LLM_SYSTEM: "Langchain",
355+
SpanAttributes.LLM_SYSTEM: vendor,
354356
SpanAttributes.LLM_TOKEN_TYPE: "output",
355357
SpanAttributes.LLM_RESPONSE_MODEL: model_name,
356358
},

‎packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/vendor_detection.py

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
from dataclasses import dataclass
2+
from typing import Set, List
3+
4+
5+
@dataclass(frozen=True)
6+
class VendorRule:
7+
exact_matches: Set[str]
8+
patterns: List[str]
9+
vendor_name: str
10+
11+
def matches(self, class_name: str) -> bool:
12+
if class_name in self.exact_matches:
13+
return True
14+
class_lower = class_name.lower()
15+
return any(pattern in class_lower for pattern in self.patterns)
16+
17+
18+
def _get_vendor_rules() -> List[VendorRule]:
19+
"""
20+
Get vendor detection rules ordered by specificity (most specific first).
21+
22+
Returns:
23+
List of VendorRule objects for detecting LLM vendors from class names
24+
"""
25+
return [
26+
VendorRule(
27+
exact_matches={"AzureChatOpenAI", "AzureOpenAI", "AzureOpenAIEmbeddings"},
28+
patterns=["azure"],
29+
vendor_name="Azure"
30+
),
31+
VendorRule(
32+
exact_matches={"ChatOpenAI", "OpenAI", "OpenAIEmbeddings"},
33+
patterns=["openai"],
34+
vendor_name="openai"
35+
),
36+
VendorRule(
37+
exact_matches={"ChatBedrock", "BedrockEmbeddings", "Bedrock", "BedrockChat"},
38+
patterns=["bedrock", "aws"],
39+
vendor_name="AWS"
40+
),
41+
VendorRule(
42+
exact_matches={"ChatAnthropic", "AnthropicLLM"},
43+
patterns=["anthropic"],
44+
vendor_name="Anthropic"
45+
),
46+
VendorRule(
47+
exact_matches={
48+
"ChatVertexAI", "VertexAI", "VertexAIEmbeddings", "ChatGoogleGenerativeAI",
49+
"GoogleGenerativeAI", "GooglePaLM", "ChatGooglePaLM"
50+
},
51+
patterns=["vertex", "google", "palm", "gemini"],
52+
vendor_name="Google"
53+
),
54+
VendorRule(
55+
exact_matches={"ChatCohere", "CohereEmbeddings", "Cohere"},
56+
patterns=["cohere"],
57+
vendor_name="Cohere"
58+
),
59+
VendorRule(
60+
exact_matches={
61+
"HuggingFacePipeline", "HuggingFaceTextGenInference",
62+
"HuggingFaceEmbeddings", "ChatHuggingFace"
63+
},
64+
patterns=["huggingface"],
65+
vendor_name="HuggingFace"
66+
),
67+
VendorRule(
68+
exact_matches={"ChatOllama", "OllamaEmbeddings", "Ollama"},
69+
patterns=["ollama"],
70+
vendor_name="Ollama"
71+
),
72+
VendorRule(
73+
exact_matches={"Together", "ChatTogether"},
74+
patterns=["together"],
75+
vendor_name="Together"
76+
),
77+
VendorRule(
78+
exact_matches={"Replicate", "ChatReplicate"},
79+
patterns=["replicate"],
80+
vendor_name="Replicate"
81+
),
82+
VendorRule(
83+
exact_matches={"ChatFireworks", "Fireworks"},
84+
patterns=["fireworks"],
85+
vendor_name="Fireworks"
86+
),
87+
VendorRule(
88+
exact_matches={"ChatGroq"},
89+
patterns=["groq"],
90+
vendor_name="Groq"
91+
),
92+
VendorRule(
93+
exact_matches={"ChatMistralAI", "MistralAI"},
94+
patterns=["mistral"],
95+
vendor_name="MistralAI"
96+
),
97+
]
98+
99+
100+
def detect_vendor_from_class(class_name: str) -> str:
101+
"""
102+
Detect vendor from LangChain model class name.
103+
Uses unified detection rules combining exact matches and patterns.
104+
105+
Args:
106+
class_name: The class name extracted from serialized model information
107+
108+
Returns:
109+
Vendor string, defaults to "Langchain" if no match found
110+
"""
111+
if not class_name:
112+
return "Langchain"
113+
114+
vendor_rules = _get_vendor_rules()
115+
116+
for rule in vendor_rules:
117+
if rule.matches(class_name):
118+
return rule.vendor_name
119+
120+
return "Langchain"

‎packages/opentelemetry-instrumentation-langchain/tests/metrics/test_langchain_metrics.py

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ def test_llm_chain_metrics(instrument_legacy, reader, chain):
4444
assert data_point.sum > 0
4545
assert (
4646
data_point.attributes[SpanAttributes.LLM_SYSTEM]
47-
== "Langchain"
47+
== "openai"
4848
)
4949

5050
if metric.name == Meters.LLM_OPERATION_DURATION:
@@ -58,7 +58,7 @@ def test_llm_chain_metrics(instrument_legacy, reader, chain):
5858
for data_point in metric.data.data_points:
5959
assert (
6060
data_point.attributes[SpanAttributes.LLM_SYSTEM]
61-
== "Langchain"
61+
== "openai"
6262
)
6363

6464
assert found_token_metric is True
@@ -96,7 +96,7 @@ def test_llm_chain_streaming_metrics(instrument_legacy, reader, llm):
9696
assert data_point.sum > 0
9797
assert (
9898
data_point.attributes[SpanAttributes.LLM_SYSTEM]
99-
== "Langchain"
99+
== "openai"
100100
)
101101

102102
if metric.name == Meters.LLM_OPERATION_DURATION:
@@ -110,7 +110,7 @@ def test_llm_chain_streaming_metrics(instrument_legacy, reader, llm):
110110
for data_point in metric.data.data_points:
111111
assert (
112112
data_point.attributes[SpanAttributes.LLM_SYSTEM]
113-
== "Langchain"
113+
== "openai"
114114
)
115115

116116
assert found_token_metric is True
@@ -124,14 +124,14 @@ def verify_token_metrics(data_points):
124124
"input",
125125
]
126126
assert data_point.sum > 0
127-
assert data_point.attributes[SpanAttributes.LLM_SYSTEM] == "Langchain"
127+
assert data_point.attributes[SpanAttributes.LLM_SYSTEM] == "openai"
128128

129129

130130
def verify_duration_metrics(data_points):
131131
assert any(data_point.count > 0 for data_point in data_points)
132132
assert any(data_point.sum > 0 for data_point in data_points)
133133
for data_point in data_points:
134-
assert data_point.attributes[SpanAttributes.LLM_SYSTEM] == "Langchain"
134+
assert data_point.attributes[SpanAttributes.LLM_SYSTEM] == "openai"
135135

136136

137137
def verify_langchain_metrics(reader):

‎packages/opentelemetry-instrumentation-langchain/tests/test_llms.py

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,7 @@ def test_custom_llm(instrument_legacy, span_exporter, log_exporter):
143143

144144
assert hugging_face_span.attributes[SpanAttributes.LLM_REQUEST_TYPE] == "completion"
145145
assert hugging_face_span.attributes[SpanAttributes.LLM_REQUEST_MODEL] == "unknown"
146+
assert hugging_face_span.attributes[SpanAttributes.LLM_SYSTEM] == "HuggingFace"
146147
assert (
147148
hugging_face_span.attributes[f"{SpanAttributes.LLM_PROMPTS}.0.content"]
148149
== "System: You are a helpful assistant\nHuman: tell me a short joke"
@@ -276,6 +277,7 @@ def test_openai(instrument_legacy, span_exporter, log_exporter):
276277

277278
assert openai_span.attributes[SpanAttributes.LLM_REQUEST_TYPE] == "chat"
278279
assert openai_span.attributes[SpanAttributes.LLM_REQUEST_MODEL] == "gpt-4o-mini"
280+
assert openai_span.attributes[SpanAttributes.LLM_SYSTEM] == "openai"
279281
assert (
280282
(openai_span.attributes[f"{SpanAttributes.LLM_PROMPTS}.0.content"])
281283
== "You are a helpful assistant"
@@ -660,6 +662,7 @@ def test_anthropic(instrument_legacy, span_exporter, log_exporter):
660662

661663
assert anthropic_span.attributes[SpanAttributes.LLM_REQUEST_TYPE] == "chat"
662664
assert anthropic_span.attributes[SpanAttributes.LLM_REQUEST_MODEL] == "claude-2.1"
665+
assert anthropic_span.attributes[SpanAttributes.LLM_SYSTEM] == "Anthropic"
663666
assert anthropic_span.attributes[SpanAttributes.LLM_REQUEST_TEMPERATURE] == 0.5
664667
assert (
665668
(anthropic_span.attributes[f"{SpanAttributes.LLM_PROMPTS}.0.content"])
@@ -870,6 +873,7 @@ def test_bedrock(instrument_legacy, span_exporter, log_exporter):
870873
bedrock_span.attributes[SpanAttributes.LLM_REQUEST_MODEL]
871874
== "anthropic.claude-3-haiku-20240307-v1:0"
872875
)
876+
assert bedrock_span.attributes[SpanAttributes.LLM_SYSTEM] == "AWS"
873877
assert (
874878
(bedrock_span.attributes[f"{SpanAttributes.LLM_PROMPTS}.0.content"])
875879
== "You are a helpful assistant"
@@ -1085,6 +1089,13 @@ def test_trace_propagation(instrument_legacy, span_exporter, log_exporter, LLM):
10851089
spans = span_exporter.get_finished_spans()
10861090
openai_span = next(span for span in spans if "OpenAI" in span.name)
10871091

1092+
expected_vendors = {
1093+
OpenAI: "openai",
1094+
VLLMOpenAI: "openai",
1095+
ChatOpenAI: "openai"
1096+
}
1097+
assert openai_span.attributes[SpanAttributes.LLM_SYSTEM] == expected_vendors[LLM]
1098+
10881099
args, kwargs = send_spy.mock.call_args
10891100
request = args[0]
10901101

0 commit comments

Comments
 (0)