diff --git a/instrumentation-genai/README.md b/instrumentation-genai/README.md index f0a90af797..e5db719917 100644 --- a/instrumentation-genai/README.md +++ b/instrumentation-genai/README.md @@ -1,6 +1,7 @@ -| Instrumentation | Supported Packages | Metrics support | Semconv status | -| --------------- | ------------------ | --------------- | -------------- | -| [opentelemetry-instrumentation-google-genai](./opentelemetry-instrumentation-google-genai) | google-genai >= 1.0.0 | No | development -| [opentelemetry-instrumentation-openai-v2](./opentelemetry-instrumentation-openai-v2) | openai >= 1.26.0 | Yes | development -| [opentelemetry-instrumentation-vertexai](./opentelemetry-instrumentation-vertexai) | google-cloud-aiplatform >= 1.64 | No | development \ No newline at end of file +| Instrumentation | Supported Packages | Metrics support | Semconv status | +|--------------------------------------------------------------------------------------------|---------------------------------|-----------------| -------------- | +| [opentelemetry-instrumentation-google-genai](./opentelemetry-instrumentation-google-genai) | google-genai >= 1.0.0 | No | development +| [opentelemetry-instrumentation-openai-v2](./opentelemetry-instrumentation-openai-v2) | openai >= 1.26.0 | Yes | development +| [opentelemetry-instrumentation-vertexai](./opentelemetry-instrumentation-vertexai) | google-cloud-aiplatform >= 1.64 | No | development +| [opentelemetry-instrumentation-langchain](./opentelemetry-instrumentation-langchain) | langchain >= 0.3.21 | Yes | development \ No newline at end of file diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/CHANGELOG.md b/instrumentation-genai/opentelemetry-instrumentation-langchain/CHANGELOG.md new file mode 100644 index 0000000000..6209a70d6f --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/CHANGELOG.md @@ -0,0 +1,8 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## Unreleased \ No newline at end of file diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/LICENSE b/instrumentation-genai/opentelemetry-instrumentation-langchain/LICENSE new file mode 100644 index 0000000000..261eeb9e9f --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/README.rst b/instrumentation-genai/opentelemetry-instrumentation-langchain/README.rst new file mode 100644 index 0000000000..e69de29bb2 diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/manual/.env b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/manual/.env new file mode 100644 index 0000000000..f136a93348 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/manual/.env @@ -0,0 +1,11 @@ +# Update this with your real OpenAI API key +OPENAI_API_KEY=sk-YOUR_API_KEY + +# Uncomment and change to your OTLP endpoint +# OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 +# OTEL_EXPORTER_OTLP_PROTOCOL=grpc + +# Change to 'false' to hide prompt and completion content +OTEL_INSTRUMENTATION_LANGCHAIN_CAPTURE_MESSAGE_CONTENT=true + +OTEL_SERVICE_NAME=opentelemetry-python-langchain \ No newline at end of file diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/manual/README.rst b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/manual/README.rst new file mode 100644 index 0000000000..b8a463cbe4 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/manual/README.rst @@ -0,0 +1,47 @@ +OpenTelemetry LangChain Instrumentation Example +============================================== + +This is an example of how to instrument LangChain calls when configuring +OpenTelemetry SDK and Instrumentations manually. + +When :code:`main.py ` is run, it exports traces, metrics (and optionally logs) +to an OTLP-compatible endpoint. Traces include details such as the span name and other attributes. +Exports metrics like input and output token usage and durations for each operation. + +Environment variables: + +- ``OTEL_INSTRUMENTATION_LANGCHAIN_CAPTURE_MESSAGE_CONTENT=true`` can be used + to capture full prompt/response content. + +Setup +----- + +1. **Update** the :code:`.env <.env>` file with any environment variables you + need (e.g., your OpenAI key, or :code:`OTEL_EXPORTER_OTLP_ENDPOINT` if not + using the default http://localhost:4317). +2. Set up a virtual environment: + + .. code-block:: console + + python3 -m venv .venv + source .venv/bin/activate + pip install "python-dotenv[cli]" + pip install -r requirements.txt + +3. **(Optional)** Install a development version of the new instrumentation: + + .. code-block:: console + + # E.g., from a local path or a git repo + pip install -e /path/to/opentelemetry-python-contrib/instrumentation-genai/opentelemetry-instrumentation-langchain +Run +--- + +Run the example like this: + +.. code-block:: console + + dotenv run -- python main.py + +You should see an example span output while traces are exported to your +configured observability tool. \ No newline at end of file diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/manual/main.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/manual/main.py new file mode 100644 index 0000000000..cbb5001d2f --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/manual/main.py @@ -0,0 +1,59 @@ +from langchain_core.messages import HumanMessage, SystemMessage +from langchain_openai import ChatOpenAI + +from opentelemetry.instrumentation.langchain import LangChainInstrumentor + +from opentelemetry import _events, _logs, trace, metrics +from opentelemetry.exporter.otlp.proto.grpc._log_exporter import ( + OTLPLogExporter, +) +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( + OTLPSpanExporter, +) +from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter + +from opentelemetry.sdk._events import EventLoggerProvider +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader + +# configure tracing +trace.set_tracer_provider(TracerProvider()) +trace.get_tracer_provider().add_span_processor( + BatchSpanProcessor(OTLPSpanExporter()) +) + +metric_reader = PeriodicExportingMetricReader(OTLPMetricExporter()) +metrics.set_meter_provider(MeterProvider(metric_readers=[metric_reader])) + +# configure logging and events +_logs.set_logger_provider(LoggerProvider()) +_logs.get_logger_provider().add_log_record_processor( + BatchLogRecordProcessor(OTLPLogExporter()) +) +_events.set_event_logger_provider(EventLoggerProvider()) + +def main(): + + # Set up instrumentation + LangChainInstrumentor().instrument() + + # ChatOpenAI + llm = ChatOpenAI(model="gpt-3.5-turbo") + messages = [ + SystemMessage(content="You are a helpful assistant!"), + HumanMessage(content="What is the capital of France?"), + ] + + result = llm.invoke(messages) + + print("LLM output:\n", result) + + # Un-instrument after use + LangChainInstrumentor().uninstrument() + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/manual/requirements.txt b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/manual/requirements.txt new file mode 100644 index 0000000000..520e1475ff --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/manual/requirements.txt @@ -0,0 +1,9 @@ +langchain==0.3.21 #todo: find the lowest compatible version +langchain_openai + +opentelemetry-sdk~=1.31.1 +opentelemetry-exporter-otlp-proto-grpc~=1.31.1 + +python-dotenv[cli] + +# For local development: `pip install -e /path/to/opentelemetry-instrumentation-langchain` \ No newline at end of file diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/zero-code/.env b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/zero-code/.env new file mode 100644 index 0000000000..f136a93348 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/zero-code/.env @@ -0,0 +1,11 @@ +# Update this with your real OpenAI API key +OPENAI_API_KEY=sk-YOUR_API_KEY + +# Uncomment and change to your OTLP endpoint +# OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 +# OTEL_EXPORTER_OTLP_PROTOCOL=grpc + +# Change to 'false' to hide prompt and completion content +OTEL_INSTRUMENTATION_LANGCHAIN_CAPTURE_MESSAGE_CONTENT=true + +OTEL_SERVICE_NAME=opentelemetry-python-langchain \ No newline at end of file diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/zero-code/README.rst b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/zero-code/README.rst new file mode 100644 index 0000000000..696a197158 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/zero-code/README.rst @@ -0,0 +1,47 @@ +OpenTelemetry LangChain Instrumentation Example +============================================== + +This is an example of how to instrument LangChain calls when configuring +OpenTelemetry SDK and Instrumentations manually. + +When :code:`main.py ` is run, it exports traces (and optionally logs) +to an OTLP-compatible endpoint. Traces include details such as the chain name, +LLM usage, token usage, and durations for each operation. + +Environment variables: + +- ``OTEL_INSTRUMENTATION_LANGCHAIN_CAPTURE_MESSAGE_CONTENT=true`` can be used + to capture full prompt/response content. + +Setup +----- + +1. **Update** the :code:`.env <.env>` file with any environment variables you + need (e.g., your OpenAI key, or :code:`OTEL_EXPORTER_OTLP_ENDPOINT` if not + using the default http://localhost:4317). +2. Set up a virtual environment: + + .. code-block:: console + + python3 -m venv .venv + source .venv/bin/activate + pip install "python-dotenv[cli]" + pip install -r requirements.txt + +3. **(Optional)** Install a development version of the new instrumentation: + + .. code-block:: console + + # E.g., from a local path or a git repo + pip install -e /path/to/opentelemetry-python-contrib/instrumentation-genai/opentelemetry-instrumentation-langchain +Run +--- + +Run the example like this: + +.. code-block:: console + + dotenv run -- opentelemetry-instrument python main.py + +You should see an example chain output while traces are exported to your +configured observability tool. \ No newline at end of file diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/zero-code/main.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/zero-code/main.py new file mode 100644 index 0000000000..c46fc6c635 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/zero-code/main.py @@ -0,0 +1,17 @@ +from langchain_core.messages import HumanMessage, SystemMessage +from langchain_openai import ChatOpenAI + +def main(): + + llm = ChatOpenAI(model="gpt-3.5-turbo") + + messages = [ + SystemMessage(content="You are a helpful assistant!"), + HumanMessage(content="What is the capital of France?"), + ] + + result = llm.invoke(messages).content + print("LLM output:\n", result) + +if __name__ == "__main__": + main() diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/zero-code/requirements.txt b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/zero-code/requirements.txt new file mode 100644 index 0000000000..c21069e4a3 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/examples/zero-code/requirements.txt @@ -0,0 +1,10 @@ +langchain==0.3.21 #todo: find the lowest compatible version +langchain_openai + +opentelemetry-sdk~=1.31.1 +opentelemetry-exporter-otlp-proto-grpc~=1.31.1 + +python-dotenv[cli] + +# For local developmen: `pip install -e /path/to/opentelemetry-instrumentation-langchain` + diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/pyproject.toml b/instrumentation-genai/opentelemetry-instrumentation-langchain/pyproject.toml new file mode 100644 index 0000000000..5a61f9a0db --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/pyproject.toml @@ -0,0 +1,56 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "opentelemetry-instrumentation-langchain" +dynamic = ["version"] +description = "OpenTelemetry Official Langchain instrumentation" +readme = "README.rst" +license = "Apache-2.0" +requires-python = ">=3.8" +authors = [ + { name = "OpenTelemetry Authors", email = "cncf-opentelemetry-contributors@lists.cncf.io" }, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", +] +dependencies = [ + "opentelemetry-api ~= 1.30", + "opentelemetry-instrumentation ~= 0.51b0", + "opentelemetry-semantic-conventions ~= 0.51b0" +] + +[project.optional-dependencies] +instruments = [ + "langchain >= 0.3.21", +] + +[project.entry-points.opentelemetry_instrumentor] +langchain = "opentelemetry.instrumentation.langchain:LangChainInstrumentor" + +[project.urls] +Homepage = "https://github.com/open-telemetry/opentelemetry-python-contrib/tree/main/instrumentation-genai/opentelemetry-instrumentation-langchain" +Repository = "https://github.com/open-telemetry/opentelemetry-python-contrib" + +[tool.hatch.version] +path = "src/opentelemetry/instrumentation/langchain/version.py" + +[tool.hatch.build.targets.sdist] +include = [ + "/src", + "/tests", +] + +[tool.hatch.build.targets.wheel] +packages = ["src/opentelemetry"] diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/__init__.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/__init__.py new file mode 100644 index 0000000000..caf8279424 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/__init__.py @@ -0,0 +1,196 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Langchain instrumentation supporting `ChatOpenAI`, it can be enabled by +using ``LangChainInstrumentor``. + +.. _langchain: https://pypi.org/project/langchain/ + +Usage +----- + +.. code:: python + + from opentelemetry.instrumentation.langchain import LangChainInstrumentor + from langchain_core.messages import HumanMessage, SystemMessage + from langchain_openai import ChatOpenAI + + LangChainInstrumentor().instrument() + + llm = ChatOpenAI(model="gpt-3.5-turbo") + messages = [ + SystemMessage(content="You are a helpful assistant!"), + HumanMessage(content="What is the capital of France?"), + ] + + result = llm.invoke(messages) + +API +--- +""" + +from typing import Collection + +from wrapt import wrap_function_wrapper + +from opentelemetry.instrumentation.langchain.config import Config +from opentelemetry.instrumentation.langchain.version import __version__ +from opentelemetry.instrumentation.langchain.package import _instruments +from opentelemetry.instrumentation.langchain.callback_handler import ( + OpenTelemetryLangChainCallbackHandler, +) +from opentelemetry.trace.propagation.tracecontext import ( + TraceContextTextMapPropagator, +) +from opentelemetry.trace import set_span_in_context +from opentelemetry.instrumentation.instrumentor import BaseInstrumentor +from opentelemetry.instrumentation.utils import unwrap +from opentelemetry.metrics import get_meter +from opentelemetry.trace import get_tracer +from opentelemetry._events import get_event_logger +from opentelemetry.semconv.schemas import Schemas + +from .instruments import Instruments + + +class LangChainInstrumentor(BaseInstrumentor): + """ + OpenTelemetry instrumentor for LangChain. + + This adds a custom callback handler to the LangChain callback manager + to capture chain, LLM, and tool events. It also wraps the internal + OpenAI invocation points (BaseChatOpenAI) to inject W3C trace headers + for downstream calls to OpenAI (or other providers). + """ + + def __init__(self, exception_logger=None, disable_trace_injection: bool = False): + """ + :param disable_trace_injection: If True, do not wrap OpenAI invocation + for trace-context injection. + """ + super().__init__() + self._disable_trace_injection = disable_trace_injection + Config.exception_logger = exception_logger + + def instrumentation_dependencies(self) -> Collection[str]: + return _instruments + + def _instrument(self, **kwargs): + tracer_provider = kwargs.get("tracer_provider") + tracer = get_tracer( + __name__, + __version__, + tracer_provider, + schema_url=Schemas.V1_28_0.value, + ) + + meter_provider = kwargs.get("meter_provider") + meter = get_meter( + __name__, + __version__, + meter_provider, + schema_url=Schemas.V1_28_0.value, + ) + + event_logger_provider = kwargs.get("event_logger_provider") + event_logger = get_event_logger( + __name__, + __version__, + event_logger_provider=event_logger_provider, + schema_url=Schemas.V1_28_0.value, + ) + + instruments = Instruments(meter) + + otel_callback_handler = OpenTelemetryLangChainCallbackHandler( + tracer=tracer, + instruments=instruments, + event_logger = event_logger, + ) + + wrap_function_wrapper( + module="langchain_core.callbacks", + name="BaseCallbackManager.__init__", + wrapper=_BaseCallbackManagerInitWrapper(otel_callback_handler), + ) + + # Optionally wrap LangChain's "BaseChatOpenAI" methods to inject trace context + if not self._disable_trace_injection: + wrap_function_wrapper( + module="langchain_openai.chat_models.base", + name="BaseChatOpenAI._generate", + wrapper=_OpenAITraceInjectionWrapper(otel_callback_handler), + ) + wrap_function_wrapper( + module="langchain_openai.chat_models.base", + name="BaseChatOpenAI._agenerate", + wrapper=_OpenAITraceInjectionWrapper(otel_callback_handler), + ) + + def _uninstrument(self, **kwargs): + """ + Cleanup instrumentation (unwrap). + """ + unwrap("langchain_core.callbacks.base", "BaseCallbackManager.__init__") + if not self._disable_trace_injection: + unwrap("langchain_openai.chat_models.base", "BaseChatOpenAI._generate") + unwrap("langchain_openai.chat_models.base", "BaseChatOpenAI._agenerate") + + +class _BaseCallbackManagerInitWrapper: + """ + Wrap the BaseCallbackManager __init__ to insert + custom callback handler in the manager's handlers list. + """ + + def __init__(self, callback_handler): + self._otel_handler = callback_handler + + def __call__(self, wrapped, instance, args, kwargs): + wrapped(*args, **kwargs) + # Ensure our OTel callback is present if not already. + for handler in instance.inheritable_handlers: + if isinstance(handler, type(self._otel_handler)): + break + else: + instance.add_handler(self._otel_handler, inherit=True) + + +class _OpenAITraceInjectionWrapper: + """ + A wrapper that intercepts calls to the underlying LLM code in LangChain + to inject W3C trace headers into upstream requests (if possible). + """ + + def __init__(self, callback_manager): + self._otel_handler = callback_manager + + def __call__(self, wrapped, instance, args, kwargs): + """ + Look up the run_id in the `kwargs["run_manager"]` to find + the active span from the callback handler. Then inject + that span context into the 'extra_headers' for the openai call. + """ + run_manager = kwargs.get("run_manager") + if run_manager is not None: + run_id = run_manager.run_id + span_holder = self._otel_handler.spans.get(run_id) + if span_holder and span_holder.span.is_recording(): + extra_headers = kwargs.get("extra_headers", {}) + ctx = set_span_in_context(span_holder.span) + TraceContextTextMapPropagator().inject(extra_headers, context=ctx) + kwargs["extra_headers"] = extra_headers + + return wrapped(*args, **kwargs) \ No newline at end of file diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py new file mode 100644 index 0000000000..f12e1f54d2 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py @@ -0,0 +1,266 @@ +import logging +import time +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Union +from uuid import UUID + +from langchain_core.callbacks import BaseCallbackHandler +from langchain_core.messages import BaseMessage +from langchain_core.outputs import LLMResult +from opentelemetry._events import EventLogger +from opentelemetry.context import get_current, Context +from opentelemetry.semconv._incubating.attributes import gen_ai_attributes as GenAI +from opentelemetry.semconv.attributes import ( + error_attributes as ErrorAttributes, +) +from opentelemetry.trace import Span, SpanKind, set_span_in_context, use_span +from opentelemetry.trace.status import Status, StatusCode + +from opentelemetry.instrumentation.langchain.config import Config +from opentelemetry.instrumentation.langchain.utils import ( + dont_throw, +) +from .instruments import Instruments +from .utils import ( + chat_generation_to_event, + message_to_event, +) + +logger = logging.getLogger(__name__) + + +@dataclass +class _SpanState: + span: Span + span_context: Context + start_time: float = field(default_factory=time.time) + request_model: Optional[str] = None + system: Optional[str] = None + children: List[UUID] = field(default_factory=list) + + +class OpenTelemetryLangChainCallbackHandler(BaseCallbackHandler): + """ + A callback handler for LangChain that uses OpenTelemetry to create spans + for chains, LLM calls, and tools. + """ + + def __init__( + self, + tracer, + instruments: Instruments, + event_logger: EventLogger, + ) -> None: + super().__init__() + self._tracer = tracer + self._duration_histogram = instruments.operation_duration_histogram + self._token_histogram = instruments.token_usage_histogram + self._event_logger = event_logger + + # Map from run_id -> _SpanState, to keep track of spans and parent/child relationships + self.spans: Dict[UUID, _SpanState] = {} + self.run_inline = True # Whether to run the callback inline. + + def _start_span( + self, + name: str, + kind: SpanKind, + parent_run_id: Optional[UUID] = None, + ) -> Span: + if parent_run_id is not None and parent_run_id in self.spans: + parent_span = self.spans[parent_run_id].span + ctx = set_span_in_context(parent_span) + span = self._tracer.start_span(name=name, kind=kind, context=ctx) + else: + # top-level or missing parent + span = self._tracer.start_span(name=name, kind=kind) + + return span + + def _end_span(self, run_id: UUID): + state = self.spans[run_id] + for child_id in state.children: + child_state = self.spans.get(child_id) + if child_state and child_state.span.end_time is None: + child_state.span.end() + if state.span.end_time is None: + state.span.end() + + def _record_duration_metric(self, run_id: UUID, request_model: Optional[str], response_model: Optional[str], operation_name: Optional[str], system: Optional[str]): + """ + Records a histogram measurement for how long the operation took. + """ + if run_id not in self.spans: + return + + elapsed = time.time() - self.spans[run_id].start_time + attributes = { + # TODO: add below to opentelemetry.semconv._incubating.attributes.gen_ai_attributes + "gen_ai.framework":"langchain", + } + if system: + attributes[GenAI.GEN_AI_SYSTEM] = system + if operation_name: + attributes[GenAI.GEN_AI_OPERATION_NAME] = operation_name + if request_model: + attributes[GenAI.GEN_AI_REQUEST_MODEL] = request_model + if response_model: + attributes[GenAI.GEN_AI_RESPONSE_MODEL] = response_model + + self._duration_histogram.record(elapsed, attributes=attributes) + + def _record_token_usage(self, token_count: int, request_model: Optional[str], response_model: Optional[str], token_type: str, operation_name: Optional[str], system: Optional[str]): + """ + Record usage of input or output tokens to a histogram. + """ + if token_count <= 0: + return + attributes = { + # TODO: add below to opentelemetry.semconv._incubating.attributes.gen_ai_attributes + "gen_ai.framework": "langchain", + GenAI.GEN_AI_TOKEN_TYPE: token_type, + } + if system: + attributes[GenAI.GEN_AI_SYSTEM] = system + if operation_name: + attributes[GenAI.GEN_AI_OPERATION_NAME] = operation_name + if request_model: + attributes[GenAI.GEN_AI_REQUEST_MODEL] = request_model + if response_model: + attributes[GenAI.GEN_AI_RESPONSE_MODEL] = response_model + + self._token_histogram.record(token_count, attributes=attributes) + + @dont_throw + def on_llm_end( + self, + response: LLMResult, + *, + run_id: UUID, + parent_run_id: Union[UUID, None] = None, + **kwargs, + ): + if Config.is_instrumentation_suppressed(): + return + + state = self.spans.get(run_id) + if not state: + return + + with use_span( + state.span, + end_on_exit=False, + ) as span: + finish_reasons = [] + for generation in getattr(response, "generations", []): + for index, chat_generation in enumerate(generation): + self._event_logger.emit(chat_generation_to_event(chat_generation, index, state.system)) + generation_info = chat_generation.generation_info + if generation_info is not None: + finish_reason = generation_info.get("finish_reason") + if finish_reason is not None: + finish_reasons.append(finish_reason or "error") + + span.set_attribute(GenAI.GEN_AI_RESPONSE_FINISH_REASONS, finish_reasons) + + response_model = None + if response.llm_output is not None: + response_model = response.llm_output.get("model_name") or response.llm_output.get("model") + if response_model is not None: + span.set_attribute(GenAI.GEN_AI_RESPONSE_MODEL, response_model) + + response_id = response.llm_output.get("id") + if response_id is not None: + span.set_attribute(GenAI.GEN_AI_RESPONSE_ID, response_id) + + # usage + usage = response.llm_output.get("usage") or response.llm_output.get("token_usage") + if usage: + prompt_tokens = usage.get("prompt_tokens", 0) + completion_tokens = usage.get("completion_tokens", 0) + span.set_attribute(GenAI.GEN_AI_USAGE_INPUT_TOKENS, prompt_tokens) + span.set_attribute(GenAI.GEN_AI_USAGE_OUTPUT_TOKENS, completion_tokens) + + # Record token usage metrics + self._record_token_usage(prompt_tokens, state.request_model, response_model, GenAI.GenAiTokenTypeValues.INPUT.value, GenAI.GenAiOperationNameValues.CHAT.value, state.system) + self._record_token_usage(completion_tokens, state.request_model, response_model, GenAI.GenAiTokenTypeValues.COMPLETION.value, GenAI.GenAiOperationNameValues.CHAT.value, state.system) + + # End the LLM span + self._end_span(run_id) + + # Record overall duration metric + self._record_duration_metric(run_id, state.request_model, response_model, GenAI.GenAiOperationNameValues.CHAT.value, state.system) + + @dont_throw + def on_chat_model_start( + self, + serialized: dict, + messages: List[List[BaseMessage]], + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs, + ): + if Config.is_instrumentation_suppressed(): + return + + system = serialized.get("name") or kwargs.get("name") or "ChatLLM" + span = self._start_span( + name=f"{system}.chat", + kind=SpanKind.CLIENT, + parent_run_id=parent_run_id, + ) + + with use_span( + span, + end_on_exit=False, + ) as span: + span.set_attribute(GenAI.GEN_AI_OPERATION_NAME, GenAI.GenAiOperationNameValues.CHAT.value) + request_model = kwargs.get("invocation_params").get("model_name") if kwargs.get("invocation_params") and kwargs.get("invocation_params").get("model_name") else None + span.set_attribute(GenAI.GEN_AI_REQUEST_MODEL, request_model) + + # TODO: add below to opentelemetry.semconv._incubating.attributes.gen_ai_attributes + span.set_attribute("gen_ai.framework", "langchain") + span.set_attribute(GenAI.GEN_AI_SYSTEM, system) + + span_state = _SpanState(span=span, span_context=get_current(), request_model=request_model, system=system) + self.spans[run_id] = span_state + + for sub_messages in messages: + for message in sub_messages: + self._event_logger.emit(message_to_event(message, system)) + + if parent_run_id is not None and parent_run_id in self.spans: + self.spans[parent_run_id].children.append(run_id) + + + @dont_throw + def on_llm_error( + self, + error: BaseException, + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs, + ): + self._handle_error(error, run_id) + + + def _handle_error(self, error: BaseException, run_id: UUID): + if Config.is_instrumentation_suppressed(): + return + state = self.spans.get(run_id) + + if not state: + return + + # Record overall duration metric + self._record_duration_metric(run_id, state.request_model, None, GenAI.GenAiOperationNameValues.CHAT.value, state.system) + + span = state.span + span.set_status(Status(StatusCode.ERROR, str(error))) + if span.is_recording(): + span.set_attribute( + ErrorAttributes.ERROR_TYPE, type(error).__qualname__ + ) + self._end_span(run_id) \ No newline at end of file diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/config.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/config.py new file mode 100644 index 0000000000..2e21ba43db --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/config.py @@ -0,0 +1,32 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class Config: + """ + Shared static config for LangChain OTel instrumentation. + """ + + # Logger to handle exceptions during instrumentation + exception_logger = None + + # Globally suppress instrumentation + _suppress_instrumentation = False + + @classmethod + def suppress_instrumentation(cls, suppress: bool = True): + cls._suppress_instrumentation = suppress + + @classmethod + def is_instrumentation_suppressed(cls) -> bool: + return cls._suppress_instrumentation diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/instruments.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/instruments.py new file mode 100644 index 0000000000..70c10055eb --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/instruments.py @@ -0,0 +1,52 @@ +from opentelemetry.metrics import Histogram, Meter +from opentelemetry.semconv._incubating.metrics import gen_ai_metrics + +_GEN_AI_CLIENT_OPERATION_DURATION_BUCKETS = [ + 0.01, + 0.02, + 0.04, + 0.08, + 0.16, + 0.32, + 0.64, + 1.28, + 2.56, + 5.12, + 10.24, + 20.48, + 40.96, + 81.92, +] + +_GEN_AI_CLIENT_TOKEN_USAGE_BUCKETS = [ + 1, + 4, + 16, + 64, + 256, + 1024, + 4096, + 16384, + 65536, + 262144, + 1048576, + 4194304, + 16777216, + 67108864, +] + + +class Instruments: + def __init__(self, meter: Meter): + self.operation_duration_histogram: Histogram = meter.create_histogram( + name=gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION, + description="GenAI operation duration", + unit="s", + explicit_bucket_boundaries_advisory=_GEN_AI_CLIENT_OPERATION_DURATION_BUCKETS, + ) + self.token_usage_histogram: Histogram = meter.create_histogram( + name=gen_ai_metrics.GEN_AI_CLIENT_TOKEN_USAGE, + description="Measures number of input and output tokens used", + unit="{token}", + explicit_bucket_boundaries_advisory=_GEN_AI_CLIENT_TOKEN_USAGE_BUCKETS, + ) diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/package.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/package.py new file mode 100644 index 0000000000..a4c4022a6e --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/package.py @@ -0,0 +1,18 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +_instruments = ( + "langchain >= 0.0.346", + "langchain-core > 0.1.0", +) diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/utils.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/utils.py new file mode 100644 index 0000000000..1bbc09a0e5 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/utils.py @@ -0,0 +1,111 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import traceback + +from opentelemetry._events import Event +from opentelemetry.semconv._incubating.attributes import gen_ai_attributes as GenAI + +logger = logging.getLogger(__name__) + +# By default, we do not record prompt or completion content. Set this +# environment variable to "true" to enable collection of message text. +OTEL_INSTRUMENTATION_LANGCHAIN_CAPTURE_MESSAGE_CONTENT = ( + "OTEL_INSTRUMENTATION_LANGCHAIN_CAPTURE_MESSAGE_CONTENT" +) + + +def should_collect_content() -> bool: + val = os.getenv(OTEL_INSTRUMENTATION_LANGCHAIN_CAPTURE_MESSAGE_CONTENT, "false") + return val.strip().lower() == "true" + + +def dont_throw(func): + """ + Decorator that catches and logs exceptions, rather than re-raising them, + to avoid interfering with user code if instrumentation fails. + """ + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except Exception as e: + logger.debug( + "OpenTelemetry instrumentation for LangChain encountered an error in %s: %s", + func.__name__, + traceback.format_exc(), + ) + from opentelemetry.instrumentation.langchain.config import Config + if Config.exception_logger: + Config.exception_logger(e) + return None + return wrapper + +def get_property_value(obj, property_name): + if isinstance(obj, dict): + return obj.get(property_name, None) + + return getattr(obj, property_name, None) + +def message_to_event(message, system): + content = get_property_value(message, "content") + if should_collect_content() and content is not None: + type = get_property_value(message, "type") + if type == "human": + type = "user" + body = {} + body["content"] = content + attributes = { + # TODO: add below to opentelemetry.semconv._incubating.attributes.gen_ai_attributes + "gen_ai.framework": "langchain", + GenAI.GEN_AI_SYSTEM: system + } + + return Event( + name=f"gen_ai.{type}.message", + attributes=attributes, + body=body if body else None, + ) + +def chat_generation_to_event(chat_generation, index, system): + if should_collect_content() and chat_generation.message: + content = get_property_value(chat_generation.message, "content") + if content is not None: + attributes = { + # TODO: add below to opentelemetry.semconv._incubating.attributes.gen_ai_attributes + "gen_ai.framework": "langchain", + GenAI.GEN_AI_SYSTEM: system + } + + finish_reason = None + generation_info = chat_generation.generation_info + if generation_info is not None: + finish_reason = generation_info.get("finish_reason") + + message = { + "content": content, + "type": chat_generation.type + } + body = { + "index": index, + "finish_reason": finish_reason or "error", + "message": message + } + + return Event( + name="gen_ai.choice", + attributes=attributes, + body=body, + ) diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/version.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/version.py new file mode 100644 index 0000000000..c9d7dd44c0 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/version.py @@ -0,0 +1,15 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__version__ = "0.0.1" \ No newline at end of file diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/__init__.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/cassettes/test_langchain_call.yaml b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/cassettes/test_langchain_call.yaml new file mode 100644 index 0000000000..381385a5f3 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/cassettes/test_langchain_call.yaml @@ -0,0 +1,144 @@ +interactions: +- request: + body: |- + { + "messages": [ + { + "content": "You are a helpful assistant!", + "role": "system" + }, + { + "content": "What is the capital of France?", + "role": "user" + } + ], + "model": "gpt-3.5-turbo", + "stream": false + } + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '171' + content-type: + - application/json + host: + - api.openai.com + traceparent: + - 00-67db16c8ff85be2c50d4dbfb5553858b-372b2c3c4b99c6d0-01 + user-agent: + - OpenAI/Python 1.86.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.86.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.13.1 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: |- + { + "id": "chatcmpl-Bj8hyoKSOooftbZZk24bce8lAT7PE", + "object": "chat.completion", + "created": 1750097934, + "model": "gpt-3.5-turbo-0125", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "The capital of France is Paris.", + "refusal": null, + "annotations": [] + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 24, + "completion_tokens": 7, + "total_tokens": 31, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "service_tier": "default", + "system_fingerprint": null + } + headers: + CF-RAY: + - 950c4ff829573a6b-LAX + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Mon, 16 Jun 2025 18:18:54 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + content-length: + - '822' + openai-organization: test_openai_org_id + openai-processing-ms: + - '381' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-envoy-upstream-service-time: + - '387' + x-ratelimit-limit-requests: + - '5000' + x-ratelimit-limit-tokens: + - '2000000' + x-ratelimit-remaining-requests: + - '4999' + x-ratelimit-remaining-tokens: + - '1999981' + x-ratelimit-reset-requests: + - 12ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_1eabd7c9c42ed2796829cbda19312189 + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/conftest.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/conftest.py new file mode 100644 index 0000000000..d9569820aa --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/conftest.py @@ -0,0 +1,237 @@ +"""Unit tests configuration module.""" + +import json +import os + +import pytest +import yaml +# from openai import AsyncOpenAI, OpenAI +from langchain_openai import ChatOpenAI + +from opentelemetry.instrumentation.langchain import LangChainInstrumentor +from opentelemetry.instrumentation.langchain.utils import ( + OTEL_INSTRUMENTATION_LANGCHAIN_CAPTURE_MESSAGE_CONTENT, +) +from opentelemetry.sdk._events import EventLoggerProvider +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk._logs.export import ( + InMemoryLogExporter, + SimpleLogRecordProcessor, +) +from opentelemetry.sdk.metrics import ( + MeterProvider, +) +from opentelemetry.sdk.metrics.export import ( + InMemoryMetricReader, +) +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) +from opentelemetry.sdk.trace.sampling import ALWAYS_OFF + + +@pytest.fixture(scope="function", name="span_exporter") +def fixture_span_exporter(): + exporter = InMemorySpanExporter() + yield exporter + + +@pytest.fixture(scope="function", name="log_exporter") +def fixture_log_exporter(): + exporter = InMemoryLogExporter() + yield exporter + + +@pytest.fixture(scope="function", name="metric_reader") +def fixture_metric_reader(): + exporter = InMemoryMetricReader() + yield exporter + + +@pytest.fixture(scope="function", name="tracer_provider") +def fixture_tracer_provider(span_exporter): + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(span_exporter)) + return provider + + +@pytest.fixture(scope="function", name="event_logger_provider") +def fixture_event_logger_provider(log_exporter): + provider = LoggerProvider() + provider.add_log_record_processor(SimpleLogRecordProcessor(log_exporter)) + event_logger_provider = EventLoggerProvider(provider) + + return event_logger_provider + + +@pytest.fixture(scope="function", name="meter_provider") +def fixture_meter_provider(metric_reader): + meter_provider = MeterProvider( + metric_readers=[metric_reader], + ) + + return meter_provider + + +@pytest.fixture(autouse=True) +def environment(): + if not os.getenv("OPENAI_API_KEY"): + os.environ["OPENAI_API_KEY"] = "test_openai_api_key" + + +@pytest.fixture +def chatOpenAI_client(): + return ChatOpenAI() + +@pytest.fixture(scope="module") +def vcr_config(): + return { + "filter_headers": [ + ("cookie", "test_cookie"), + ("authorization", "Bearer test_openai_api_key"), + ("openai-organization", "test_openai_org_id"), + ("openai-project", "test_openai_project_id"), + ], + "decode_compressed_response": True, + "before_record_response": scrub_response_headers, + } + + +@pytest.fixture(scope="function") +def instrument_no_content( + tracer_provider, event_logger_provider, meter_provider +): + os.environ.update( + {OTEL_INSTRUMENTATION_LANGCHAIN_CAPTURE_MESSAGE_CONTENT: "False"} + ) + + instrumentor = LangChainInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + event_logger_provider=event_logger_provider, + meter_provider=meter_provider, + ) + + yield instrumentor + os.environ.pop(OTEL_INSTRUMENTATION_LANGCHAIN_CAPTURE_MESSAGE_CONTENT, None) + instrumentor.uninstrument() + + +@pytest.fixture(scope="function") +def instrument_with_content( + tracer_provider, event_logger_provider, meter_provider +): + os.environ.update( + {OTEL_INSTRUMENTATION_LANGCHAIN_CAPTURE_MESSAGE_CONTENT: "True"} + ) + instrumentor = LangChainInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + event_logger_provider=event_logger_provider, + meter_provider=meter_provider, + ) + + yield instrumentor + os.environ.pop(OTEL_INSTRUMENTATION_LANGCHAIN_CAPTURE_MESSAGE_CONTENT, None) + instrumentor.uninstrument() + + +@pytest.fixture(scope="function") +def instrument_with_content_unsampled( + span_exporter, event_logger_provider, meter_provider +): + os.environ.update( + {OTEL_INSTRUMENTATION_LANGCHAIN_CAPTURE_MESSAGE_CONTENT: "True"} + ) + + tracer_provider = TracerProvider(sampler=ALWAYS_OFF) + tracer_provider.add_span_processor(SimpleSpanProcessor(span_exporter)) + + instrumentor = LangChainInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + event_logger_provider=event_logger_provider, + meter_provider=meter_provider, + ) + + yield instrumentor + os.environ.pop(OTEL_INSTRUMENTATION_LANGCHAIN_CAPTURE_MESSAGE_CONTENT, None) + instrumentor.uninstrument() + + +class LiteralBlockScalar(str): + """Formats the string as a literal block scalar, preserving whitespace and + without interpreting escape characters""" + + +def literal_block_scalar_presenter(dumper, data): + """Represents a scalar string as a literal block, via '|' syntax""" + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|") + + +yaml.add_representer(LiteralBlockScalar, literal_block_scalar_presenter) + + +def process_string_value(string_value): + """Pretty-prints JSON or returns long strings as a LiteralBlockScalar""" + try: + json_data = json.loads(string_value) + return LiteralBlockScalar(json.dumps(json_data, indent=2)) + except (ValueError, TypeError): + if len(string_value) > 80: + return LiteralBlockScalar(string_value) + return string_value + + +def convert_body_to_literal(data): + """Searches the data for body strings, attempting to pretty-print JSON""" + if isinstance(data, dict): + for key, value in data.items(): + # Handle response body case (e.g., response.body.string) + if key == "body" and isinstance(value, dict) and "string" in value: + value["string"] = process_string_value(value["string"]) + + # Handle request body case (e.g., request.body) + elif key == "body" and isinstance(value, str): + data[key] = process_string_value(value) + + else: + convert_body_to_literal(value) + + elif isinstance(data, list): + for idx, choice in enumerate(data): + data[idx] = convert_body_to_literal(choice) + + return data + + +class PrettyPrintJSONBody: + """This makes request and response body recordings more readable.""" + + @staticmethod + def serialize(cassette_dict): + cassette_dict = convert_body_to_literal(cassette_dict) + return yaml.dump( + cassette_dict, default_flow_style=False, allow_unicode=True + ) + + @staticmethod + def deserialize(cassette_string): + return yaml.load(cassette_string, Loader=yaml.Loader) + + +@pytest.fixture(scope="module", autouse=True) +def fixture_vcr(vcr): + vcr.register_serializer("yaml", PrettyPrintJSONBody) + return vcr + + +def scrub_response_headers(response): + """ + This scrubs sensitive response headers. Note they are case-sensitive! + """ + response["headers"]["openai-organization"] = "test_openai_org_id" + response["headers"]["Set-Cookie"] = "test_set_cookie" + return response diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_langchain_llm.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_langchain_llm.py new file mode 100644 index 0000000000..829331f262 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_langchain_llm.py @@ -0,0 +1,221 @@ +from langchain_core.messages import HumanMessage, SystemMessage +from langchain_openai import ChatOpenAI + +import pytest +from typing import Optional + +from opentelemetry.sdk.trace import ReadableSpan + +from opentelemetry.semconv._incubating.attributes import ( + event_attributes as EventAttributes, +) + +from opentelemetry.semconv._incubating.metrics import gen_ai_metrics +from opentelemetry.semconv._incubating.attributes import gen_ai_attributes + + +# span_exporter, log_exporter, chatOpenAI_client, instrument_no_content are coming from +# fixtures defined in conftest.py +@pytest.mark.vcr() +def test_langchain_call( + span_exporter, log_exporter, metric_reader, chatOpenAI_client, instrument_with_content +): + llm_model_value = "gpt-3.5-turbo" + llm = ChatOpenAI(model=llm_model_value) + + messages = [ + SystemMessage(content="You are a helpful assistant!"), + HumanMessage(content="What is the capital of France?"), + ] + + response = llm.invoke(messages) + assert response.content == "The capital of France is Paris." + + # verify spans + spans = span_exporter.get_finished_spans() + print(f"spans: {spans}") + for span in spans: + print(f"span: {span}") + print(f"span attributes: {span.attributes}") + # TODO: fix the code and ensure the assertions are correct + assert_openai_completion_attributes(spans[0], llm_model_value, response) + + # verify logs + logs = log_exporter.get_finished_logs() + print(f"logs: {logs}") + for log in logs: + print(f"log: {log}") + print(f"log attributes: {log.log_record.attributes}") + print(f"log body: {log.log_record.body}") + system_message = {"content": messages[0].content} + human_message = {"content": messages[1].content} + assert len(logs) == 3 + assert_message_in_logs( + logs[0], "gen_ai.system.message", system_message, spans[0] + ) + assert_message_in_logs( + logs[1], "gen_ai.user.message", human_message, spans[0] + ) + + chat_generation_event = { + "index": 0, + "finish_reason": "stop", + "message": { + "content": response.content, + "type": "ChatGeneration" + } + } + assert_message_in_logs(logs[2], "gen_ai.choice", chat_generation_event, spans[0]) + + # verify metrics + metrics = metric_reader.get_metrics_data().resource_metrics + print(f"metrics: {metrics}") + assert len(metrics) == 1 + + metric_data = metrics[0].scope_metrics[0].metrics + for m in metric_data: + if m.name == gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION: + assert_duration_metric(m, spans[0]) + if m.name == gen_ai_metrics.GEN_AI_CLIENT_TOKEN_USAGE: + assert_token_usage_metric(m, spans[0]) + +def assert_openai_completion_attributes( + span: ReadableSpan, + request_model: str, + response: Optional, + operation_name: str = "chat", +): + return assert_all_openai_attributes( + span, + request_model, + response.response_metadata.get("model_name"), + response.response_metadata.get("token_usage").get("prompt_tokens"), + response.response_metadata.get("token_usage").get("completion_tokens"), + operation_name, + ) + +def assert_all_openai_attributes( + span: ReadableSpan, + request_model: str, + response_model: str = "gpt-3.5-turbo-0125", + input_tokens: Optional[int] = None, + output_tokens: Optional[int] = None, + operation_name: str = "chat", + span_name: str = "ChatOpenAI.chat", + system: str = "ChatOpenAI", + framework: str = "langchain", +): + assert span.name == span_name + assert operation_name == span.attributes[gen_ai_attributes.GEN_AI_OPERATION_NAME] + assert framework == span.attributes["gen_ai.framework"] + assert system == span.attributes[gen_ai_attributes.GEN_AI_SYSTEM] + assert request_model == "gpt-3.5-turbo" + assert response_model == "gpt-3.5-turbo-0125" + assert gen_ai_attributes.GEN_AI_RESPONSE_ID in span.attributes + + if input_tokens: + assert ( + input_tokens + == span.attributes[gen_ai_attributes.GEN_AI_USAGE_INPUT_TOKENS] + ) + else: + assert gen_ai_attributes.GEN_AI_USAGE_INPUT_TOKENS not in span.attributes + + if output_tokens: + assert ( + output_tokens + == span.attributes[gen_ai_attributes.GEN_AI_USAGE_OUTPUT_TOKENS] + ) + else: + assert ( + gen_ai_attributes.GEN_AI_USAGE_OUTPUT_TOKENS not in span.attributes + ) + +def assert_message_in_logs(log, event_name, expected_content, parent_span): + assert log.log_record.attributes[EventAttributes.EVENT_NAME] == event_name + assert ( + # TODO: use constant from GenAIAttributes.GenAiSystemValues after it is added there + log.log_record.attributes[gen_ai_attributes.GEN_AI_SYSTEM] + == "ChatOpenAI" + ) + + if not expected_content: + assert not log.log_record.body + else: + assert log.log_record.body + assert dict(log.log_record.body) == remove_none_values( + expected_content + ) + assert_log_parent(log, parent_span) + +def remove_none_values(body): + result = {} + for key, value in body.items(): + if value is None: + continue + if isinstance(value, dict): + result[key] = remove_none_values(value) + elif isinstance(value, list): + result[key] = [remove_none_values(i) for i in value] + else: + result[key] = value + return result + +def assert_log_parent(log, span): + if span: + assert log.log_record.trace_id == span.get_span_context().trace_id + assert log.log_record.span_id == span.get_span_context().span_id + assert ( + log.log_record.trace_flags == span.get_span_context().trace_flags + ) + +def assert_duration_metric(metric, parent_span): + assert metric is not None + assert len(metric.data.data_points) == 1 + assert metric.data.data_points[0].sum > 0 + + assert_duration_metric_attributes(metric.data.data_points[0].attributes, parent_span) + assert_exemplars(metric.data.data_points[0].exemplars, metric.data.data_points[0].sum, parent_span) + +def assert_duration_metric_attributes(attributes, parent_span): + assert len(attributes) == 5 + assert attributes.get("gen_ai.framework") == "langchain" + assert attributes.get(gen_ai_attributes.GEN_AI_SYSTEM) == "ChatOpenAI" + assert attributes.get(gen_ai_attributes.GEN_AI_OPERATION_NAME) == gen_ai_attributes.GenAiOperationNameValues.CHAT.value + assert attributes.get(gen_ai_attributes.GEN_AI_REQUEST_MODEL) == parent_span.attributes[ + gen_ai_attributes.GEN_AI_REQUEST_MODEL + ] + assert attributes.get(gen_ai_attributes.GEN_AI_RESPONSE_MODEL) == parent_span.attributes[ + gen_ai_attributes.GEN_AI_RESPONSE_MODEL + ] + +def assert_token_usage_metric(metric, parent_span): + assert metric is not None + assert len(metric.data.data_points) == 2 + + assert metric.data.data_points[0].sum > 0 + assert_token_usage_metric_attributes(metric.data.data_points[0].attributes, parent_span) + assert_exemplars(metric.data.data_points[0].exemplars, metric.data.data_points[0].sum, parent_span) + + assert metric.data.data_points[1].sum > 0 + assert_token_usage_metric_attributes(metric.data.data_points[1].attributes, parent_span) + assert_exemplars(metric.data.data_points[1].exemplars, metric.data.data_points[1].sum, parent_span) + +def assert_token_usage_metric_attributes(attributes, parent_span): + assert len(attributes) == 6 + assert attributes.get("gen_ai.framework") == "langchain" + assert attributes.get(gen_ai_attributes.GEN_AI_SYSTEM) == "ChatOpenAI" + assert attributes.get(gen_ai_attributes.GEN_AI_OPERATION_NAME) == gen_ai_attributes.GenAiOperationNameValues.CHAT.value + assert attributes.get(gen_ai_attributes.GEN_AI_REQUEST_MODEL) == parent_span.attributes[ + gen_ai_attributes.GEN_AI_REQUEST_MODEL + ] + assert attributes.get(gen_ai_attributes.GEN_AI_RESPONSE_MODEL) == parent_span.attributes[ + gen_ai_attributes.GEN_AI_RESPONSE_MODEL + ] + +def assert_exemplars(exemplars, sum, parent_span): + assert len(exemplars) == 1 + assert exemplars[0].value == sum + assert exemplars[0].span_id == parent_span.get_span_context().span_id + assert exemplars[0].trace_id == parent_span.get_span_context().trace_id +