diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/CHANGELOG.md b/instrumentation-genai/opentelemetry-instrumentation-langchain/CHANGELOG.md index 6209a70d6f..c3d51f9fef 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/CHANGELOG.md +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/CHANGELOG.md @@ -5,4 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Unreleased \ No newline at end of file +## Unreleased + +- Added span support for genAI langchain llm invocation. + ([#3665](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3665)) \ No newline at end of file diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/manual/.env b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/manual/.env new file mode 100644 index 0000000000..e34c860897 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/manual/.env @@ -0,0 +1,8 @@ +# Update this with your real OpenAI API key +OPENAI_API_KEY=sk-YOUR_API_KEY + +# Uncomment and change to your OTLP endpoint +# OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 +# OTEL_EXPORTER_OTLP_PROTOCOL=grpc + +OTEL_SERVICE_NAME=opentelemetry-python-langchain-manual diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/manual/README.rst b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/manual/README.rst new file mode 100644 index 0000000000..2c829bc801 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/manual/README.rst @@ -0,0 +1,39 @@ +OpenTelemetry Langcahin Instrumentation Example +============================================ + +This is an example of how to instrument Langchain when configuring OpenTelemetry SDK and instrumentations manually. + +When :code:`main.py `_ is run, it exports traces to an OTLP-compatible endpoint. +Traces include details such as the span name and other attributes. + +Note: :code:`.env <.env>`_ file configures additional environment variables: +- :code:`OTEL_LOGS_EXPORTER=otlp` to specify exporter type. +- :code:`OPENAI_API_KEY` open AI key for accessing the OpenAI API. +- :code:`OTEL_EXPORTER_OTLP_ENDPOINT` to specify the endpoint for exporting traces (default is http://localhost:4317). + +Setup +----- + +Minimally, update the :code:`.env <.env>`_ file with your :code:`OPENAI_API_KEY`. +An OTLP compatible endpoint should be listening for traces http://localhost:4317. +If not, update :code:`OTEL_EXPORTER_OTLP_ENDPOINT` as well. + +Next, set up a virtual environment like this: + +:: + + python3 -m venv .venv + source .venv/bin/activate + pip install "python-dotenv[cli]" + pip install -r requirements.txt + +Run +--- + +Run the example like this: + +:: + + dotenv run -- python main.py + +You should see the capital of France generated by Langchain ChatOpenAI while traces export to your configured observability tool. diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/manual/main.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/manual/main.py new file mode 100644 index 0000000000..e18f53ff31 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/manual/main.py @@ -0,0 +1,48 @@ +from langchain_core.messages import HumanMessage, SystemMessage +from langchain_openai import ChatOpenAI + +from opentelemetry import trace +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( + OTLPSpanExporter, +) +from opentelemetry.instrumentation.langchain import LangChainInstrumentor +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor + +# Configure tracing +trace.set_tracer_provider(TracerProvider()) +span_processor = BatchSpanProcessor(OTLPSpanExporter()) +trace.get_tracer_provider().add_span_processor(span_processor) + + +def main(): + # Set up instrumentation + LangChainInstrumentor().instrument() + + # ChatOpenAI + llm = ChatOpenAI( + model="gpt-3.5-turbo", + temperature=0.1, + max_tokens=100, + top_p=0.9, + frequency_penalty=0.5, + presence_penalty=0.5, + stop_sequences=["\n", "Human:", "AI:"], + seed=100, + ) + + messages = [ + SystemMessage(content="You are a helpful assistant!"), + HumanMessage(content="What is the capital of France?"), + ] + + result = llm.invoke(messages) + + print("LLM output:\n", result) + + # Un-instrument after use + LangChainInstrumentor().uninstrument() + + +if __name__ == "__main__": + main() diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/manual/requirements.txt b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/manual/requirements.txt new file mode 100644 index 0000000000..af55da6c94 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/manual/requirements.txt @@ -0,0 +1,7 @@ +langchain==0.3.21 +langchain_openai +opentelemetry-sdk~=1.36.0 +opentelemetry-exporter-otlp-proto-grpc~=1.36.0 + +# Uncomment after lanchain instrumetation is released +# opentelemetry-instrumentation-langchain~=2.0b0.dev \ No newline at end of file diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/zero-code/.env b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/zero-code/.env new file mode 100644 index 0000000000..b5f56bb3fd --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/zero-code/.env @@ -0,0 +1,8 @@ +# Update this with your real OpenAI API key +OPENAI_API_KEY=sk-YOUR_API_KEY + +# Uncomment and change to your OTLP endpoint +# OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 +# OTEL_EXPORTER_OTLP_PROTOCOL=grpc + +OTEL_SERVICE_NAME=opentelemetry-python-langchain-zero-code \ No newline at end of file diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/zero-code/README.rst b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/zero-code/README.rst new file mode 100644 index 0000000000..3d141ed033 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/zero-code/README.rst @@ -0,0 +1,40 @@ +OpenTelemetry Langchain Zero-Code Instrumentation Example +====================================================== + +This is an example of how to instrument Langchain with zero code changes, +using `opentelemetry-instrument`. + +When :code:`main.py `_ is run, it exports traces to an OTLP-compatible endpoint. +Traces include details such as the span name and other attributes. + +Note: :code:`.env <.env>`_ file configures additional environment variables: +- :code:`OTEL_LOGS_EXPORTER=otlp` to specify exporter type. +- :code:`OPENAI_API_KEY` open AI key for accessing the OpenAI API. +- :code:`OTEL_EXPORTER_OTLP_ENDPOINT` to specify the endpoint for exporting traces (default is http://localhost:4317). + +Setup +----- + +Minimally, update the :code:`.env <.env>`_ file with your :code:`OPENAI_API_KEY`. +An OTLP compatible endpoint should be listening for traces http://localhost:4317. +If not, update :code:`OTEL_EXPORTER_OTLP_ENDPOINT` as well. + +Next, set up a virtual environment like this: + +:: + + python3 -m venv .venv + source .venv/bin/activate + pip install "python-dotenv[cli]" + pip install -r requirements.txt + +Run +--- + +Run the example like this: + +:: + + dotenv run -- opentelemetry-instrument python main.py + +You should see the capital of France generated by Langchain ChatOpenAI while traces export to your configured observability tool. diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/zero-code/main.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/zero-code/main.py new file mode 100644 index 0000000000..24ee6db16d --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/zero-code/main.py @@ -0,0 +1,27 @@ +from langchain_core.messages import HumanMessage, SystemMessage +from langchain_openai import ChatOpenAI + + +def main(): + llm = ChatOpenAI( + model="gpt-3.5-turbo", + temperature=0.1, + max_tokens=100, + top_p=0.9, + frequency_penalty=0.5, + presence_penalty=0.5, + stop_sequences=["\n", "Human:", "AI:"], + seed=100, + ) + + messages = [ + SystemMessage(content="You are a helpful assistant!"), + HumanMessage(content="What is the capital of France?"), + ] + + result = llm.invoke(messages).content + print("LLM output:\n", result) + + +if __name__ == "__main__": + main() diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/zero-code/requirements.txt b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/zero-code/requirements.txt new file mode 100644 index 0000000000..59caa5c61f --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/zero-code/requirements.txt @@ -0,0 +1,8 @@ +langchain==0.3.21 +langchain_openai +opentelemetry-sdk~=1.36.0 +opentelemetry-exporter-otlp-proto-grpc~=1.36.0 +opentelemetry-distro~=0.57b0 + +# Uncomment after lanchain instrumetation is released +# opentelemetry-instrumentation-langchain~=2.0b0.dev diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/pyproject.toml b/instrumentation-genai/opentelemetry-instrumentation-langchain/pyproject.toml index 55e24185f2..dad30f6740 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/pyproject.toml +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/pyproject.toml @@ -25,9 +25,9 @@ classifiers = [ "Programming Language :: Python :: 3.13", ] dependencies = [ - "opentelemetry-api ~= 1.30", - "opentelemetry-instrumentation ~= 0.51b0", - "opentelemetry-semantic-conventions ~= 0.51b0" + "opentelemetry-api >= 1.36.0", + "opentelemetry-instrumentation >= 0.57b0", + "opentelemetry-semantic-conventions >= 0.57b0" ] [project.optional-dependencies] @@ -35,6 +35,9 @@ instruments = [ "langchain >= 0.3.21", ] +[project.entry-points.opentelemetry_instrumentor] +langchain = "opentelemetry.instrumentation.langchain:LangChainInstrumentor" + [project.urls] Homepage = "https://github.com/open-telemetry/opentelemetry-python-contrib/tree/main/instrumentation-genai/opentelemetry-instrumentation-langchain" Repository = "https://github.com/open-telemetry/opentelemetry-python-contrib" diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/__init__.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/__init__.py index e69de29bb2..aca1d22f58 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/__init__.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/__init__.py @@ -0,0 +1,121 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Langchain instrumentation supporting `ChatOpenAI`, it can be enabled by +using ``LangChainInstrumentor``. + +Usage +----- +.. code:: python + from opentelemetry.instrumentation.langchain import LangChainInstrumentor + from langchain_core.messages import HumanMessage, SystemMessage + from langchain_openai import ChatOpenAI + + LangChainInstrumentor().instrument() + llm = ChatOpenAI(model="gpt-3.5-turbo", temperature=0, max_tokens=1000) + messages = [ + SystemMessage(content="You are a helpful assistant!"), + HumanMessage(content="What is the capital of France?"), + ] + result = llm.invoke(messages) + LangChainInstrumentor().uninstrument() + +API +--- +""" + +from typing import Any, Callable, Collection, Optional + +from wrapt import wrap_function_wrapper # type: ignore + +from opentelemetry.instrumentation.instrumentor import BaseInstrumentor +from opentelemetry.instrumentation.langchain.callback_handler import ( + OpenTelemetryLangChainCallbackHandler, +) +from opentelemetry.instrumentation.langchain.package import _instruments +from opentelemetry.instrumentation.langchain.version import __version__ +from opentelemetry.instrumentation.utils import unwrap +from opentelemetry.semconv.schemas import Schemas +from opentelemetry.trace import get_tracer + + +class LangChainInstrumentor(BaseInstrumentor): + """ + OpenTelemetry instrumentor for LangChain. + This adds a custom callback handler to the LangChain callback manager + to capture LLM telemetry. + """ + + def __init__( + self, exception_logger: Optional[Callable[[Exception], Any]] = None + ): + super().__init__() + + def instrumentation_dependencies(self) -> Collection[str]: + return _instruments + + def _instrument(self, **kwargs: Any): + """ + Enable Langchain instrumentation. + """ + tracer_provider = kwargs.get("tracer_provider") + tracer = get_tracer( + __name__, + __version__, + tracer_provider, + schema_url=Schemas.V1_28_0.value, + ) + + otel_callback_handler = OpenTelemetryLangChainCallbackHandler( + tracer=tracer, + ) + + wrap_function_wrapper( + module="langchain_core.callbacks", + name="BaseCallbackManager.__init__", + wrapper=_BaseCallbackManagerInitWrapper(otel_callback_handler), + ) + + def _uninstrument(self, **kwargs: Any): + """ + Cleanup instrumentation (unwrap). + """ + unwrap("langchain_core.callbacks.base.BaseCallbackManager", "__init__") + + +class _BaseCallbackManagerInitWrapper: + """ + Wrap the BaseCallbackManager __init__ to insert custom callback handler in the manager's handlers list. + """ + + def __init__( + self, callback_handler: OpenTelemetryLangChainCallbackHandler + ): + self._otel_handler = callback_handler + + def __call__( + self, + wrapped: Callable[..., None], + instance: Any, + args: tuple[Any, ...], + kwargs: dict[str, Any], + ): + wrapped(*args, **kwargs) + # Ensure our OTel callback is present if not already. + for handler in instance.inheritable_handlers: + if isinstance(handler, type(self._otel_handler)): + break + else: + instance.add_handler(self._otel_handler, inherit=True) diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py new file mode 100644 index 0000000000..1f41662e86 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py @@ -0,0 +1,172 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any, Dict, List, Optional +from uuid import UUID + +from langchain_core.callbacks import BaseCallbackHandler # type: ignore +from langchain_core.messages import BaseMessage # type: ignore +from langchain_core.outputs import LLMResult # type: ignore + +from opentelemetry.instrumentation.langchain.span_manager import _SpanManager +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAI, +) +from opentelemetry.trace import Tracer + + +class OpenTelemetryLangChainCallbackHandler(BaseCallbackHandler): # type: ignore[misc] + """ + A callback handler for LangChain that uses OpenTelemetry to create spans for LLM calls and chains, tools etc,. in future. + """ + + def __init__( + self, + tracer: Tracer, + ) -> None: + super().__init__() # type: ignore + + self.span_manager = _SpanManager( + tracer=tracer, + ) + + def on_chat_model_start( + self, + serialized: Dict[str, Any], + messages: List[List[BaseMessage]], # type: ignore + *, + run_id: UUID, + tags: Optional[List[str]] = None, + parent_run_id: Optional[UUID] = None, + metadata: Optional[Dict[str, Any]] = None, + **kwargs: Any, + ) -> None: + invocation_params = kwargs.get("invocation_params") + request_model = ( + invocation_params.get("model_name") if invocation_params else "" + ) + span = self.span_manager.create_llm_span( + run_id=run_id, + parent_run_id=parent_run_id, + request_model=request_model, + ) + + if invocation_params is not None: + top_p = invocation_params.get("top_p") + if top_p is not None: + span.set_attribute(GenAI.GEN_AI_REQUEST_TOP_P, top_p) + frequency_penalty = invocation_params.get("frequency_penalty") + if frequency_penalty is not None: + span.set_attribute( + GenAI.GEN_AI_REQUEST_FREQUENCY_PENALTY, frequency_penalty + ) + presence_penalty = invocation_params.get("presence_penalty") + if presence_penalty is not None: + span.set_attribute( + GenAI.GEN_AI_REQUEST_PRESENCE_PENALTY, presence_penalty + ) + stop_sequences = invocation_params.get("stop") + if stop_sequences is not None: + span.set_attribute( + GenAI.GEN_AI_REQUEST_STOP_SEQUENCES, stop_sequences + ) + seed = invocation_params.get("seed") + if seed is not None: + span.set_attribute(GenAI.GEN_AI_REQUEST_SEED, seed) + + if metadata is not None: + max_tokens = metadata.get("ls_max_tokens") + if max_tokens is not None: + span.set_attribute(GenAI.GEN_AI_REQUEST_MAX_TOKENS, max_tokens) + provider = metadata.get("ls_provider") + if provider is not None: + # TODO: add to semantic conventions + span.set_attribute("gen_ai.provider.name", provider) + temperature = metadata.get("ls_temperature") + if temperature is not None: + span.set_attribute( + GenAI.GEN_AI_REQUEST_TEMPERATURE, temperature + ) + + def on_llm_end( + self, + response: LLMResult, # type: ignore + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> None: + span = self.span_manager.get_span(run_id) + + if span is None: + # If the span does not exist, we cannot set attributes or end it + return + + finish_reasons: List[str] = [] + for generation in getattr(response, "generations", []): # type: ignore + for chat_generation in generation: + generation_info = getattr( + chat_generation, "generation_info", None + ) + if generation_info is not None: + finish_reason = generation_info.get("finish_reason") + if finish_reason is not None: + finish_reasons.append(str(finish_reason) or "error") + + span.set_attribute( + GenAI.GEN_AI_RESPONSE_FINISH_REASONS, finish_reasons + ) + + llm_output = getattr(response, "llm_output", None) # type: ignore + if llm_output is not None: + response_model = llm_output.get("model_name") or llm_output.get( + "model" + ) + if response_model is not None: + span.set_attribute( + GenAI.GEN_AI_RESPONSE_MODEL, str(response_model) + ) + + response_id = llm_output.get("id") + if response_id is not None: + span.set_attribute(GenAI.GEN_AI_RESPONSE_ID, str(response_id)) + + # usage + usage = llm_output.get("usage") or llm_output.get("token_usage") + if usage: + prompt_tokens = usage.get("prompt_tokens", 0) + completion_tokens = usage.get("completion_tokens", 0) + span.set_attribute( + GenAI.GEN_AI_USAGE_INPUT_TOKENS, + int(prompt_tokens) if prompt_tokens is not None else 0, + ) + span.set_attribute( + GenAI.GEN_AI_USAGE_OUTPUT_TOKENS, + int(completion_tokens) + if completion_tokens is not None + else 0, + ) + + # End the LLM span + self.span_manager.end_span(run_id) + + def on_llm_error( + self, + error: BaseException, + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> None: + self.span_manager.handle_error(error, run_id) diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/span_manager.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/span_manager.py new file mode 100644 index 0000000000..2122a13c07 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/span_manager.py @@ -0,0 +1,116 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass, field +from typing import Dict, List, Optional +from uuid import UUID + +from opentelemetry.context import Context, get_current +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAI, +) +from opentelemetry.semconv.attributes import ( + error_attributes as ErrorAttributes, +) +from opentelemetry.trace import Span, SpanKind, Tracer, set_span_in_context +from opentelemetry.trace.status import Status, StatusCode + +__all__ = ["_SpanManager"] + + +@dataclass +class _SpanState: + span: Span + context: Context + children: List[UUID] = field(default_factory=list) + + +class _SpanManager: + def __init__( + self, + tracer: Tracer, + ) -> None: + self._tracer = tracer + + # Map from run_id -> _SpanState, to keep track of spans and parent/child relationships + self.spans: Dict[UUID, _SpanState] = {} + + def _create_span( + self, + run_id: UUID, + parent_run_id: Optional[UUID], + span_name: str, + kind: SpanKind = SpanKind.INTERNAL, + ) -> Span: + if parent_run_id is not None and parent_run_id in self.spans: + parent_state = self.spans[parent_run_id] + parent_span = parent_state.span + ctx = set_span_in_context(parent_span) + span = self._tracer.start_span( + name=span_name, kind=kind, context=ctx + ) + parent_state.children.append(run_id) + else: + # top-level or missing parent + span = self._tracer.start_span(name=span_name, kind=kind) + + span_state = _SpanState(span=span, context=get_current()) + self.spans[run_id] = span_state + + return span + + def create_llm_span( + self, + run_id: UUID, + parent_run_id: Optional[UUID], + request_model: str, + ) -> Span: + span = self._create_span( + run_id=run_id, + parent_run_id=parent_run_id, + span_name=f"{GenAI.GenAiOperationNameValues.CHAT.value} {request_model}", + kind=SpanKind.CLIENT, + ) + span.set_attribute( + GenAI.GEN_AI_OPERATION_NAME, + GenAI.GenAiOperationNameValues.CHAT.value, + ) + span.set_attribute(GenAI.GEN_AI_REQUEST_MODEL, request_model) + + return span + + def end_span(self, run_id: UUID) -> None: + state = self.spans[run_id] + for child_id in state.children: + child_state = self.spans.get(child_id) + if child_state: + # Always end child spans as OpenTelemetry spans don't expose end_time directly + child_state.span.end() + # Always end the span as OpenTelemetry spans don't expose end_time directly + state.span.end() + + def get_span(self, run_id: UUID) -> Optional[Span]: + state = self.spans.get(run_id) + return state.span if state else None + + def handle_error(self, error: BaseException, run_id: UUID): + span = self.get_span(run_id) + if span is None: + # If the span does not exist, we cannot set the error status + return + span.set_status(Status(StatusCode.ERROR, str(error))) + span.set_attribute( + ErrorAttributes.ERROR_TYPE, type(error).__qualname__ + ) + self.end_span(run_id) diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/cassettes/test_langchain_call.yaml b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/cassettes/test_langchain_call.yaml new file mode 100644 index 0000000000..a21d51258b --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/cassettes/test_langchain_call.yaml @@ -0,0 +1,157 @@ +interactions: +- request: + body: |- + { + "messages": [ + { + "content": "You are a helpful assistant!", + "role": "system" + }, + { + "content": "What is the capital of France?", + "role": "user" + } + ], + "model": "gpt-3.5-turbo", + "frequency_penalty": 0.5, + "max_completion_tokens": 100, + "presence_penalty": 0.5, + "seed": 100, + "stop": [ + "\n", + "Human:", + "AI:" + ], + "stream": false, + "temperature": 0.1, + "top_p": 0.9 + } + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '316' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.98.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.98.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.13.5 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: |- + { + "id": "chatcmpl-BzXyZbwSwCmuSOKwRx7tnsia3V9lH", + "object": "chat.completion", + "created": 1754008311, + "model": "gpt-3.5-turbo-0125", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "The capital of France is Paris.", + "refusal": null, + "annotations": [] + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 24, + "completion_tokens": 7, + "total_tokens": 31, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "service_tier": "default", + "system_fingerprint": null + } + headers: + CF-RAY: + - 96813c201ae62f46-LAX + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Fri, 01 Aug 2025 00:31:51 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + content-length: + - '822' + openai-organization: test_openai_org_id + openai-processing-ms: + - '189' + openai-project: + - proj_GLiYlAc06hF0Fm06IMReZLy4 + openai-version: + - '2020-10-01' + x-envoy-decorator-operation: + - router.openai.svc.cluster.local:5004/* + x-envoy-upstream-service-time: + - '211' + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '200000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '199982' + x-ratelimit-reset-requests: + - 8.64s + x-ratelimit-reset-tokens: + - 5ms + x-request-id: + - 8a5db9a5-6103-41d4-baf3-813be646bae4 + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/conftest.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/conftest.py new file mode 100644 index 0000000000..e5a0931407 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/conftest.py @@ -0,0 +1,142 @@ +"""Unit tests configuration module.""" + +import json +import os + +import pytest +import yaml +from langchain_openai import ChatOpenAI + +from opentelemetry.instrumentation.langchain import LangChainInstrumentor +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) + + +@pytest.fixture(scope="function", name="span_exporter") +def fixture_span_exporter(): + exporter = InMemorySpanExporter() + yield exporter + + +@pytest.fixture(scope="function", name="tracer_provider") +def fixture_tracer_provider(span_exporter): + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(span_exporter)) + return provider + + +@pytest.fixture(scope="function") +def start_instrumentation( + tracer_provider, +): + instrumentor = LangChainInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + ) + + yield instrumentor + instrumentor.uninstrument() + + +@pytest.fixture(autouse=True) +def environment(): + if not os.getenv("OPENAI_API_KEY"): + os.environ["OPENAI_API_KEY"] = "test_openai_api_key" + + +@pytest.fixture +def chatOpenAI_client(): + return ChatOpenAI() + + +@pytest.fixture(scope="module") +def vcr_config(): + return { + "filter_headers": [ + ("cookie", "test_cookie"), + ("authorization", "Bearer test_openai_api_key"), + ("openai-organization", "test_openai_org_id"), + ("openai-project", "test_openai_project_id"), + ], + "decode_compressed_response": True, + "before_record_response": scrub_response_headers, + } + + +class LiteralBlockScalar(str): + """Formats the string as a literal block scalar, preserving whitespace and + without interpreting escape characters""" + + +def literal_block_scalar_presenter(dumper, data): + """Represents a scalar string as a literal block, via '|' syntax""" + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|") + + +yaml.add_representer(LiteralBlockScalar, literal_block_scalar_presenter) + + +def process_string_value(string_value): + """Pretty-prints JSON or returns long strings as a LiteralBlockScalar""" + try: + json_data = json.loads(string_value) + return LiteralBlockScalar(json.dumps(json_data, indent=2)) + except (ValueError, TypeError): + if len(string_value) > 80: + return LiteralBlockScalar(string_value) + return string_value + + +def convert_body_to_literal(data): + """Searches the data for body strings, attempting to pretty-print JSON""" + if isinstance(data, dict): + for key, value in data.items(): + # Handle response body case (e.g., response.body.string) + if key == "body" and isinstance(value, dict) and "string" in value: + value["string"] = process_string_value(value["string"]) + + # Handle request body case (e.g., request.body) + elif key == "body" and isinstance(value, str): + data[key] = process_string_value(value) + + else: + convert_body_to_literal(value) + + elif isinstance(data, list): + for idx, choice in enumerate(data): + data[idx] = convert_body_to_literal(choice) + + return data + + +class PrettyPrintJSONBody: + """This makes request and response body recordings more readable.""" + + @staticmethod + def serialize(cassette_dict): + cassette_dict = convert_body_to_literal(cassette_dict) + return yaml.dump( + cassette_dict, default_flow_style=False, allow_unicode=True + ) + + @staticmethod + def deserialize(cassette_string): + return yaml.load(cassette_string, Loader=yaml.Loader) + + +@pytest.fixture(scope="module", autouse=True) +def fixture_vcr(vcr): + vcr.register_serializer("yaml", PrettyPrintJSONBody) + return vcr + + +def scrub_response_headers(response): + """ + This scrubs sensitive response headers. Note they are case-sensitive! + """ + response["headers"]["openai-organization"] = "test_openai_org_id" + response["headers"]["Set-Cookie"] = "test_set_cookie" + return response diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_llm_call.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_llm_call.py new file mode 100644 index 0000000000..6b4c173ab6 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_llm_call.py @@ -0,0 +1,101 @@ +from typing import Optional + +import pytest +from langchain_core.messages import HumanMessage, SystemMessage +from langchain_openai import ChatOpenAI + +from opentelemetry.sdk.trace import ReadableSpan +from opentelemetry.semconv._incubating.attributes import gen_ai_attributes + + +# span_exporter, chatOpenAI_client, start_instrumentation are coming from fixtures defined in conftest.py +@pytest.mark.vcr() +def test_langchain_call( + span_exporter, chatOpenAI_client, start_instrumentation +): + llm = ChatOpenAI( + model="gpt-3.5-turbo", + temperature=0.1, + max_tokens=100, + top_p=0.9, + frequency_penalty=0.5, + presence_penalty=0.5, + stop_sequences=["\n", "Human:", "AI:"], + seed=100, + ) + + messages = [ + SystemMessage(content="You are a helpful assistant!"), + HumanMessage(content="What is the capital of France?"), + ] + + response = llm.invoke(messages) + assert response.content == "The capital of France is Paris." + + # verify spans + spans = span_exporter.get_finished_spans() + print(f"spans: {spans}") + for span in spans: + print(f"span: {span}") + print(f"span attributes: {span.attributes}") + assert_openai_completion_attributes(spans[0], response) + + +def assert_openai_completion_attributes( + span: ReadableSpan, response: Optional +): + assert span.name == "chat gpt-3.5-turbo" + assert span.attributes[gen_ai_attributes.GEN_AI_OPERATION_NAME] == "chat" + assert ( + span.attributes[gen_ai_attributes.GEN_AI_REQUEST_MODEL] + == "gpt-3.5-turbo" + ) + assert ( + span.attributes[gen_ai_attributes.GEN_AI_RESPONSE_MODEL] + == "gpt-3.5-turbo-0125" + ) + assert span.attributes[gen_ai_attributes.GEN_AI_REQUEST_MAX_TOKENS] == 100 + assert span.attributes[gen_ai_attributes.GEN_AI_REQUEST_TEMPERATURE] == 0.1 + # TODO: add to semantic conventions + assert span.attributes["gen_ai.provider.name"] == "openai" + assert gen_ai_attributes.GEN_AI_RESPONSE_ID in span.attributes + assert span.attributes[gen_ai_attributes.GEN_AI_REQUEST_TOP_P] == 0.9 + assert ( + span.attributes[gen_ai_attributes.GEN_AI_REQUEST_FREQUENCY_PENALTY] + == 0.5 + ) + assert ( + span.attributes[gen_ai_attributes.GEN_AI_REQUEST_PRESENCE_PENALTY] + == 0.5 + ) + stop_sequences = span.attributes.get( + gen_ai_attributes.GEN_AI_REQUEST_STOP_SEQUENCES + ) + assert all(seq in ["\n", "Human:", "AI:"] for seq in stop_sequences) + assert span.attributes[gen_ai_attributes.GEN_AI_REQUEST_SEED] == 100 + + input_tokens = response.response_metadata.get("token_usage").get( + "prompt_tokens" + ) + if input_tokens: + assert ( + input_tokens + == span.attributes[gen_ai_attributes.GEN_AI_USAGE_INPUT_TOKENS] + ) + else: + assert ( + gen_ai_attributes.GEN_AI_USAGE_INPUT_TOKENS not in span.attributes + ) + + output_tokens = response.response_metadata.get("token_usage").get( + "completion_tokens" + ) + if output_tokens: + assert ( + output_tokens + == span.attributes[gen_ai_attributes.GEN_AI_USAGE_OUTPUT_TOKENS] + ) + else: + assert ( + gen_ai_attributes.GEN_AI_USAGE_OUTPUT_TOKENS not in span.attributes + ) diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_span_manager.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_span_manager.py new file mode 100644 index 0000000000..2643740a60 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_span_manager.py @@ -0,0 +1,84 @@ +import unittest.mock +import uuid + +import pytest + +from opentelemetry.instrumentation.langchain.span_manager import ( + SpanManager, + _SpanState, +) +from opentelemetry.trace import SpanKind, get_tracer +from opentelemetry.trace.span import Span + + +class TestSpanManager: + @pytest.fixture + def tracer(self): + return get_tracer("test_tracer") + + @pytest.fixture + def handler(self, tracer): + return SpanManager(tracer=tracer) + + @pytest.mark.parametrize( + "parent_run_id,parent_in_spans", + [ + (None, False), # No parent + (uuid.uuid4(), False), # Parent not in spans + (uuid.uuid4(), True), # Parent in spans + ], + ) + def test_create_span( + self, handler, tracer, parent_run_id, parent_in_spans + ): + # Arrange + run_id = uuid.uuid4() + span_name = "test_span" + kind = SpanKind.INTERNAL + + mock_span = unittest.mock.Mock(spec=Span) + + # Setup parent if needed + if parent_run_id is not None and parent_in_spans: + parent_mock_span = unittest.mock.Mock(spec=Span) + handler.spans[parent_run_id] = _SpanState( + span=parent_mock_span, context=None + ) + + with ( + unittest.mock.patch.object( + tracer, "start_span", return_value=mock_span + ) as mock_start_span, + unittest.mock.patch( + "opentelemetry.instrumentation.langchain.span_manager.set_span_in_context" + ) as mock_set_span_in_context, + unittest.mock.patch( + "opentelemetry.instrumentation.langchain.span_manager.get_current" + ), + ): + # Act + result = handler.create_span( + run_id, parent_run_id, span_name, kind + ) + + # Assert + assert result == mock_span + assert run_id in handler.spans + assert handler.spans[run_id].span == mock_span + + # Verify parent-child relationship + if parent_run_id is not None and parent_in_spans: + mock_set_span_in_context.assert_called_once_with( + handler.spans[parent_run_id].span + ) + mock_start_span.assert_called_once_with( + name=span_name, + kind=kind, + context=mock_set_span_in_context.return_value, + ) + assert run_id in handler.spans[parent_run_id].children + else: + mock_start_span.assert_called_once_with( + name=span_name, kind=kind + ) + mock_set_span_in_context.assert_not_called()