diff --git a/docs/instrumentation-genai/langchain/langchain.rst b/docs/instrumentation-genai/langchain/langchain.rst new file mode 100644 index 0000000000..c0fd2be194 --- /dev/null +++ b/docs/instrumentation-genai/langchain/langchain.rst @@ -0,0 +1,7 @@ +OpenTelemetry Jinja2 Instrumentation +==================================== + +.. automodule:: opentelemetry.instrumentation.langchain + :members: + :undoc-members: + :show-inheritance: diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/__init__.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/__init__.py index e69de29bb2..f08ada8c91 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/__init__.py +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/__init__.py @@ -0,0 +1,59 @@ +from typing import Collection +from opentelemetry.instrumentation.instrumentor import BaseInstrumentor +from wrapt import wrap_function_wrapper + +from opentelemetry.trace import get_tracer +from opentelemetry.instrumentation.utils import unwrap + +# from opentelemetry.instrumentation.langchain_v2.version import __version__ +from opentelemetry.instrumentation.langchain.version import __version__ +from opentelemetry.instrumentation.langchain_v2.callback_handler import OpenTelemetryCallbackHandler +from opentelemetry.instrumentation.langchain_v2.async_callback_handler import OpenTelemetryAsyncCallbackHandler + + +__all__ = ["OpenTelemetryCallbackHandler"] + +_instruments = ("langchain >= 0.1.0",) + +class LangChainInstrumentor(BaseInstrumentor): + + def instrumentation_dependencies(cls) -> Collection[str]: + return _instruments + + def _instrument(self, **kwargs): + tracer_provider = kwargs.get("tracer_provider") + tracer = get_tracer(__name__, __version__, tracer_provider) + + otelCallbackHandler = OpenTelemetryCallbackHandler(tracer) + + wrap_function_wrapper( + module="langchain_core.callbacks", + name="BaseCallbackManager.__init__", + wrapper=_BaseCallbackManagerInitWrapper(otelCallbackHandler), + ) + + def _uninstrument(self, **kwargs): + unwrap("langchain_core.callbacks", "BaseCallbackManager.__init__") + if hasattr(self, "_wrapped"): + for module, name in self._wrapped: + unwrap(module, name) + self.handler = None + +class _BaseCallbackManagerInitWrapper: + def __init__(self, callback_handler: "OpenTelemetryCallbackHandler"): + self.callback_handler = callback_handler + self._wrapped = [] + + def __call__( + self, + wrapped, + instance, + args, + kwargs, + ) -> None: + wrapped(*args, **kwargs) + for handler in instance.inheritable_handlers: + if isinstance(handler, OpenTelemetryCallbackHandler): + return None + else: + instance.add_handler(self.callback_handler, True) \ No newline at end of file diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py new file mode 100644 index 0000000000..165d3fa87b --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/callback_handler.py @@ -0,0 +1,496 @@ +import time +from dataclasses import dataclass, field +from typing import Any, Optional +from langchain_core.callbacks import ( + BaseCallbackHandler, +) + +from langchain_core.messages import BaseMessage +from langchain_core.outputs import LLMResult +from opentelemetry.trace import SpanKind, set_span_in_context +from opentelemetry.trace.span import Span +from opentelemetry.util.types import AttributeValue +from uuid import UUID + +from opentelemetry import context as context_api +from opentelemetry.instrumentation.utils import _SUPPRESS_INSTRUMENTATION_KEY + +from langchain_core.agents import AgentAction, AgentFinish + +from opentelemetry.instrumentation.langchain.span_attributes import Span_Attributes, GenAIOperationValues +from opentelemetry.trace.status import Status, StatusCode + +@dataclass +class SpanHolder: + span: Span + children: list[UUID] + start_time: float = field(default_factory=time.time()) + request_model: Optional[str] = None + +def _set_request_params(span, kwargs, span_holder: SpanHolder): + + for model_tag in ("model_id", "base_model_id"): + if (model := kwargs.get(model_tag)) is not None: + span_holder.request_model = model + break + elif ( + model := (kwargs.get("invocation_params") or {}).get(model_tag) + ) is not None: + span_holder.request_model = model + break + else: + model = "unknown" + + if span_holder.request_model is None: + model = None + + _set_span_attribute(span, Span_Attributes.GEN_AI_REQUEST_MODEL, model) + _set_span_attribute(span, Span_Attributes.GEN_AI_RESPONSE_MODEL, model) + + if "invocation_params" in kwargs: + params = ( + kwargs["invocation_params"].get("params") or kwargs["invocation_params"] + ) + else: + params = kwargs + + _set_span_attribute( + span, + Span_Attributes.GEN_AI_REQUEST_MAX_TOKENS, + params.get("max_tokens") or params.get("max_new_tokens"), + ) + + _set_span_attribute( + span, Span_Attributes.GEN_AI_REQUEST_TEMPERATURE, params.get("temperature") + ) + + _set_span_attribute(span, Span_Attributes.GEN_AI_REQUEST_TOP_P, params.get("top_p")) + + +def _set_span_attribute(span: Span, name: str, value: AttributeValue): + if value is not None and value != "": + span.set_attribute(name, value) + + +def _sanitize_metadata_value(value: Any) -> Any: + """Convert metadata values to OpenTelemetry-compatible types.""" + if value is None: + return None + if isinstance(value, (bool, str, bytes, int, float)): + return value + if isinstance(value, (list, tuple)): + return [str(_sanitize_metadata_value(v)) for v in value] + return str(value) + +class OpenTelemetryCallbackHandler(BaseCallbackHandler): + def __init__(self, tracer): + super().__init__() + self.tracer = tracer + self.span_mapping: dict[UUID, SpanHolder] = {} + + + def _end_span(self, span: Span, run_id: UUID) -> None: + for child_id in self.span_mapping[run_id].children: + child_span = self.span_mapping[child_id].span + if child_span.end_time is None: + child_span.end() + span.end() + + + def _create_span( + self, + run_id: UUID, + parent_run_id: Optional[UUID], + span_name: str, + kind: SpanKind = SpanKind.INTERNAL, + metadata: Optional[dict[str, Any]] = None, + ) -> Span: + + metadata = metadata or {} + + if metadata is not None: + current_association_properties = ( + context_api.get_value("association_properties") or {} + ) + sanitized_metadata = { + k: _sanitize_metadata_value(v) + for k, v in metadata.items() + if v is not None + } + context_api.attach( + context_api.set_value( + "association_properties", + {**current_association_properties, **sanitized_metadata}, + ) + ) + + if parent_run_id is not None and parent_run_id in self.span_mapping: + span = self.tracer.start_span( + span_name, + context=set_span_in_context(self.span_mapping[parent_run_id].span), + kind=kind, + ) + else: + span = self.tracer.start_span(span_name, kind=kind) + _set_span_attribute(span, "root_span", True) + + model_id = "unknown" + + if "invocation_params" in metadata: + if "base_model_id" in metadata["invocation_params"]: + model_id = metadata["invocation_params"]["base_model_id"] + elif "model_id" in metadata["invocation_params"]: + model_id = metadata["invocation_params"]["model_id"] + + self.span_mapping[run_id] = SpanHolder( + span, [], time.time(), model_id + ) + + if parent_run_id is not None and parent_run_id in self.span_mapping: + self.span_mapping[parent_run_id].children.append(run_id) + + return span + + + @staticmethod + def _get_name_from_callback( + serialized: dict[str, Any], + _tags: Optional[list[str]] = None, + _metadata: Optional[dict[str, Any]] = None, + **kwargs: Any, + ) -> str: + """Get the name to be used for the span. Based on heuristic. Can be extended.""" + if serialized and "kwargs" in serialized and serialized["kwargs"].get("name"): + return serialized["kwargs"]["name"] + if kwargs.get("name"): + return kwargs["name"] + if serialized.get("name"): + return serialized["name"] + if "id" in serialized: + return serialized["id"][-1] + + return "unknown" + + + def _handle_error( + self, + error: BaseException, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> None: + """Common error handling logic for all components.""" + if context_api.get_value(_SUPPRESS_INSTRUMENTATION_KEY): + return + + span = self.span_mapping[run_id].span + span.set_status(Status(StatusCode.ERROR)) + span.record_exception(error) + self._end_span(span, run_id) + + + def on_chat_model_start(self, + serialized: dict[str, Any], + messages: list[list[BaseMessage]], + *, + run_id: UUID, + tags: Optional[list[str]] = None, + parent_run_id: Optional[UUID] = None, + metadata: Optional[dict[str, Any]] = None, + **kwargs: Any + ): + + if context_api.get_value(_SUPPRESS_INSTRUMENTATION_KEY): + return + model_id = None + if "invocation_params" in kwargs and "model_id" in kwargs["invocation_params"]: + model_id = kwargs["invocation_params"]["model_id"] + + name = self._get_name_from_callback(serialized, kwargs=kwargs) + if model_id != None: + name = model_id + + span = self._create_span( + run_id, + parent_run_id, + f"{GenAIOperationValues.CHAT} {name}", + kind=SpanKind.CLIENT, + metadata=metadata, + ) + _set_span_attribute(span, Span_Attributes.GEN_AI_OPERATION_NAME, GenAIOperationValues.CHAT) + + + if "kwargs" in serialized: + _set_request_params(span, serialized["kwargs"], self.span_mapping[run_id]) + if "name" in serialized: + _set_span_attribute(span, Span_Attributes.GEN_AI_SYSTEM, serialized.get("name")) + _set_span_attribute(span, Span_Attributes.GEN_AI_OPERATION_NAME, "chat") + + + def on_llm_start(self, + serialized: dict[str, Any], + prompts: list[str], + *, + run_id: UUID, + parent_run_id: UUID | None = None, + tags: Optional[list[str]] | None = None, + metadata: Optional[dict[str,Any]] | None = None, + **kwargs: Any + ): + if context_api.get_value(_SUPPRESS_INSTRUMENTATION_KEY): + return + + model_id = None + if "invocation_params" in kwargs and "model_id" in kwargs["invocation_params"]: + model_id = kwargs["invocation_params"]["model_id"] + + name = self._get_name_from_callback(serialized, kwargs=kwargs) + if model_id != None: + name = model_id + + span = self._create_span( + run_id, + parent_run_id, + f"{GenAIOperationValues.CHAT} {name}", + kind=SpanKind.CLIENT, + metadata=metadata, + ) + _set_span_attribute(span, Span_Attributes.GEN_AI_OPERATION_NAME, GenAIOperationValues.CHAT) + + _set_request_params(span, kwargs, self.span_mapping[run_id]) + + _set_span_attribute(span, Span_Attributes.GEN_AI_SYSTEM, serialized.get("name")) + + _set_span_attribute(span, Span_Attributes.GEN_AI_OPERATION_NAME, "text_completion") + + + def on_llm_end(self, + response: LLMResult, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + tags: Optional[list[str]] | None = None, + **kwargs: Any + ): + if context_api.get_value(_SUPPRESS_INSTRUMENTATION_KEY): + return + + span = None + if run_id in self.span_mapping: + span = self.span_mapping[run_id].span + else: + return + + model_name = None + if response.llm_output is not None: + model_name = response.llm_output.get( + "model_name" + ) or response.llm_output.get("model_id") + if model_name is not None: + _set_span_attribute(span, Span_Attributes.GEN_AI_RESPONSE_MODEL, model_name) + + id = response.llm_output.get("id") + if id is not None and id != "": + _set_span_attribute(span, Span_Attributes.GEN_AI_RESPONSE_ID, id) + + token_usage = (response.llm_output or {}).get("token_usage") or ( + response.llm_output or {} + ).get("usage") + + if token_usage is not None: + prompt_tokens = ( + token_usage.get("prompt_tokens") + or token_usage.get("input_token_count") + or token_usage.get("input_tokens") + ) + completion_tokens = ( + token_usage.get("completion_tokens") + or token_usage.get("generated_token_count") + or token_usage.get("output_tokens") + ) + + _set_span_attribute( + span, Span_Attributes.GEN_AI_USAGE_INPUT_TOKENS, prompt_tokens + ) + + _set_span_attribute( + span, Span_Attributes.GEN_AI_USAGE_OUTPUT_TOKENS, completion_tokens + ) + + self._end_span(span, run_id) + + def on_llm_error(self, + error: BaseException, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + tags: Optional[list[str]] | None = None, + **kwargs: Any + ): + self._handle_error(error, run_id, parent_run_id, **kwargs) + + + def on_chain_start(self, + serialized: dict[str, Any], + inputs: dict[str, Any], + *, + run_id: UUID, + parent_run_id: UUID | None = None, + tags: Optional[list[str]] | None = None, + metadata: Optional[dict[str,Any]] | None = None, + **kwargs: Any + ): + if context_api.get_value(_SUPPRESS_INSTRUMENTATION_KEY): + return + + + name = self._get_name_from_callback(serialized, **kwargs) + + span_name = f"chain {name}" + span = self._create_span( + run_id, + parent_run_id, + span_name, + metadata=metadata, + ) + + if "agent_name" in metadata: + _set_span_attribute(span, Span_Attributes.GEN_AI_AGENT_NAME, metadata["agent_name"]) + + _set_span_attribute(span, "gen_ai.prompt", str(inputs)) + + + def on_chain_end(self, + outputs: dict[str, Any], + *, + run_id: UUID, + parent_run_id: UUID | None = None, + tags: list[str] | None = None, + **kwargs: Any + ): + + if context_api.get_value(_SUPPRESS_INSTRUMENTATION_KEY): + return + + span_holder = self.span_mapping[run_id] + span = span_holder.span + _set_span_attribute(span, "gen_ai.completion", str(outputs)) + self._end_span(span, run_id) + + + def on_chain_error(self, + error: BaseException, + run_id: UUID, + parent_run_id: UUID | None = None, + tags: Optional[list[str]] | None = None, + **kwargs: Any + ): + self._handle_error(error, run_id, parent_run_id, **kwargs) + + + def on_tool_start(self, + serialized: dict[str, Any], + input_str: str, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + tags: list[str] | None = None, + metadata: dict[str, Any] | None = None, + inputs: dict[str, Any] | None = None, + **kwargs: Any + ): + if context_api.get_value(_SUPPRESS_INSTRUMENTATION_KEY): + return + + + name = self._get_name_from_callback(serialized, kwargs=kwargs) + span_name = f"execute_tool {name}" + span = self._create_span( + run_id, + parent_run_id, + span_name, + metadata=metadata, + ) + + _set_span_attribute(span, "gen_ai.tool.input", input_str) + + if serialized.get("id"): + _set_span_attribute( + span, + Span_Attributes.GEN_AI_TOOL_CALL_ID, + serialized.get("id") + ) + + if serialized.get("description"): + _set_span_attribute( + span, + Span_Attributes.GEN_AI_TOOL_DESCRIPTION, + serialized.get("description"), + ) + + _set_span_attribute( + span, + Span_Attributes.GEN_AI_TOOL_NAME, + name + ) + + _set_span_attribute(span, Span_Attributes.GEN_AI_OPERATION_NAME, "execute_tool") + + + def on_tool_end(self, + output: Any, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + tags: list[str] | None = None, + **kwargs: Any + ): + if context_api.get_value(_SUPPRESS_INSTRUMENTATION_KEY): + return + + span = self.span_mapping[run_id].span + + _set_span_attribute(span, "gen_ai.tool.output", str(output)) + self._end_span(span, run_id) + + + def on_tool_error(self, + error: BaseException, + run_id: UUID, + parent_run_id: UUID| None = None, + tags: list[str] | None = None, + **kwargs: Any, + ): + self._handle_error(error, run_id, parent_run_id, **kwargs) + + + def on_agent_action(self, + action: AgentAction, + run_id: UUID, + parent_run_id: UUID, + **kwargs: Any + ): + tool = getattr(action, "tool", None) + tool_input = getattr(action, "tool_input", None) + + if run_id in self.span_mapping: + span = self.span_mapping[run_id].span + + _set_span_attribute(span, "gen_ai.agent.tool.input", tool_input) + _set_span_attribute(span, "gen_ai.agent.tool.name", tool) + _set_span_attribute(span, Span_Attributes.GEN_AI_OPERATION_NAME, "invoke_agent") + + def on_agent_finish(self, + finish: AgentFinish, + run_id: UUID, + parent_run_id: UUID, + **kwargs: Any + ): + + span = self.span_mapping[run_id].span + + _set_span_attribute(span, "gen_ai.agent.tool.output", finish.return_values['output']) + + + def on_agent_error(self, error, run_id, parent_run_id, **kwargs): + self._handle_error(error, run_id, parent_run_id, **kwargs) \ No newline at end of file diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/span_attributes.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/span_attributes.py new file mode 100644 index 0000000000..4422d9cec4 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/span_attributes.py @@ -0,0 +1,41 @@ +""" +Semantic conventions for Gen AI agent spans following OpenTelemetry standards. + +This module defines constants for span attribute names as specified in: +https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/gen-ai-agent-spans.md +""" + +class Span_Attributes: + GEN_AI_OPERATION_NAME = "gen_ai.operation.name" + GEN_AI_SYSTEM = "gen_ai.system" + GEN_AI_ERROR_TYPE = "error.type" + GEN_AI_AGENT_DESCRIPTION = "gen_ai.agent.description" + GEN_AI_AGENT_ID = "gen_ai.agent.id" + GEN_AI_AGENT_NAME = "gen_ai.agent.name" + GEN_AI_REQUEST_MODEL = "gen_ai.request.model" + GEN_AI_SERVER_PORT = "server.port" + GEN_AI_REQUEST_FREQUENCY_PENALTY = "gen_ai.request.frequency_penalty" + GEN_AI_REQUEST_MAX_TOKENS = "gen_ai.request.max_tokens" + GEN_AI_REQUEST_PRESENCE_PENALTY = "gen_ai.request.presence_penalty" + GEN_AI_REQUEST_TEMPERATURE = "gen_ai.request.temperature" + GEN_AI_REQUEST_TOP_P = "gen_ai.request.top_p" + GEN_AI_RESPONSE_FINISH_REASONS = "gen_ai.response.finish_reasons" + GEN_AI_RESPONSE_ID = "gen_ai.response.id" + GEN_AI_RESPONSE_MODEL = "gen_ai.response.model" + GEN_AI_USAGE_INPUT_TOKENS = "gen_ai.usage.input_tokens" + GEN_AI_USAGE_OUTPUT_TOKENS = "gen_ai.usage.output_tokens" + GEN_AI_SERVER_ADDR = "server.address" + GEN_AI_TOOL_CALL_ID= "gen_ai.tool.call.id" + GEN_AI_TOOL_NAME = "gen_ai.tool.name" + GEN_AI_TOOL_DESCRIPTION = "gen_ai.tool.description" + GEN_AI_TOOL_TYPE = "gen_ai.tool.type" + + +class GenAIOperationValues: + CHAT = "chat" + CREATE_AGENT = "create_agent" + EMBEDDINGS = "embeddings" + GENERATE_CONTENT = "generate_content" + INVOKE_AGENT = "invoke_agent" + TEXT_COMPLETION = "text_completion" + UNKNOWN = "unknown" \ No newline at end of file diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/utils.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/utils.py new file mode 100644 index 0000000000..82c8b79fc4 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/src/opentelemetry/instrumentation/langchain/utils.py @@ -0,0 +1,3 @@ +OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT = ( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT" +) \ No newline at end of file diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/conftest.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/conftest.py new file mode 100644 index 0000000000..2eadfa6000 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/conftest.py @@ -0,0 +1,182 @@ +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + + +from opentelemetry.sdk._events import EventLoggerProvider +from opentelemetry.sdk._logs.export import ( + InMemoryLogExporter, + SimpleLogRecordProcessor, +) + +from opentelemetry.sdk.metrics import ( + MeterProvider, +) +from opentelemetry.sdk.metrics.export import ( + InMemoryMetricReader, +) + +from opentelemetry.instrumentation.langchain import LangChainInstrumentor + +@pytest.fixture(scope="function", name="span_exporter") +def fixture_span_exporter(): + exporter = InMemorySpanExporter() + yield exporter + + +@pytest.fixture(scope="function", name="log_exporter") +def fixture_log_exporter(): + exporter = InMemoryLogExporter() + yield exporter + + +@pytest.fixture(scope="function", name="metric_reader") +def fixture_metric_reader(): + exporter = InMemoryMetricReader() + yield exporter + + +@pytest.fixture(scope="function", name="tracer_provider") +def fixture_tracer_provider(span_exporter): + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(span_exporter)) + return provider + + +# # not sure if needed +# @pytest.fixture(scope="function", name="event_logger_provider") +# def fixture_event_logger_provider(log_exporter): +# provider = LoggerProvider() +# provider.add_log_record_processor(SimpleLogRecordProcessor(log_exporter)) +# event_logger_provider = EventLoggerProvider(provider) + +# return event_logger_provider + + +# # not sure if needed +# @pytest.fixture(scope="function", name="meter_provider") +# def fixture_meter_provider(metric_reader): +# meter_provider = MeterProvider( +# metric_readers=[metric_reader], +# ) + +# return meter_provider + + +@pytest.fixture(autouse=True) +def environment(): + + if not os.getenv("AWS_ACCESS_KEY_ID"): + os.environ["AWS_ACCESS_KEY_ID"] = "test_aws_access_key_id" + + if not os.getenv("AWS_SECRET_ACCESS_KEY"): + os.environ["AWS_SECRET_ACCESS_KEY"] = "test_aws_secret_access_key" + + if not os.getenv("AWS_REGION"): + os.environ["AWS_REGION"] = "us-west-2" + + if not os.getenv("AWS_BEDROCK_ENDPOINT_URL"): + os.environ["AWS_BEDROCK_ENDPOINT_URL"] = "https://bedrock.us-west-2.amazonaws.com" + + if not os.getenv("AWS_PROFILE"): + os.environ["AWS_PROFILE"] = "default" + + + + +def scrub_aws_credentials(response): + """Remove sensitive data from response headers.""" + if "headers" in response: + for sensitive_header in [ + "x-amz-security-token", + "x-amz-request-id", + "x-amzn-requestid", + "x-amz-id-2" + ]: + if sensitive_header in response["headers"]: + response["headers"][sensitive_header] = ["REDACTED"] + return response + +@pytest.fixture(scope="module") +def vcr_config(): + return { + "filter_headers": [ + ("authorization", "AWS4-HMAC-SHA256 REDACTED"), + ("x-amz-date", "REDACTED_DATE"), + ("x-amz-security-token", "REDACTED_TOKEN"), + ("x-amz-content-sha256", "REDACTED_CONTENT_HASH"), + ], + "filter_query_parameters": [ + ("X-Amz-Security-Token", "REDACTED"), + ("X-Amz-Signature", "REDACTED"), + ], + "decode_compressed_response": True, + "before_record_response": scrub_aws_credentials, + } + +@pytest.fixture(scope="session") +def instrument_langchain(reader, tracer_provider): + langchain_instrumentor = LangChainInstrumentor() + langchain_instrumentor.instrument( + tracer_provider=tracer_provider + ) + + yield + + langchain_instrumentor.uninstrument() + +@pytest.fixture(scope="function") +def instrument_no_content( + tracer_provider +): + os.environ.update( + {OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT: "False"} + ) + + instrumentor = LangChainInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + ) + yield instrumentor + os.environ.pop(OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT, None) + instrumentor.uninstrument() + + +@pytest.fixture(scope="function") +def instrument_with_content( + tracer_provider +): + os.environ.update( + {OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT: "True"} + ) + instrumentor = LangChainInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider + ) + + yield instrumentor + os.environ.pop(OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT, None) + instrumentor.uninstrument() + + +@pytest.fixture(scope="function") +def instrument_with_content_unsampled( + span_exporter +): + os.environ.update( + {OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT: "True"} + ) + + tracer_provider = TracerProvider(sampler=ALWAYS_OFF) + tracer_provider.add_span_processor(SimpleSpanProcessor(span_exporter)) + + instrumentor = LangChainInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider + ) + + yield instrumentor + os.environ.pop(OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT, None) + instrumentor.uninstrument() + diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test-requirements.txt b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test-requirements.txt new file mode 100644 index 0000000000..f9824ca282 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test-requirements.txt @@ -0,0 +1,67 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# ******************************** +# WARNING: NOT HERMETIC !!!!!!!!!! +# ******************************** +# +# This "requirements.txt" is installed in conjunction +# with multiple other dependencies in the top-level "tox.ini" +# file. In particular, please see: +# +# openai-latest: {[testenv]test_deps} +# openai-latest: -r {toxinidir}/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/requirements.latest.txt +# +# This provides additional dependencies, namely: +# +# opentelemetry-api +# opentelemetry-sdk +# opentelemetry-semantic-conventions +# +# ... with a "dev" version based on the latest distribution. + + +# This variant of the requirements aims to test the system using +# the newest supported version of external dependencies. + + +# LangChain and related packages +langchain +langchain-aws +langchain-community + +# AWS +boto3 + +# Agent tools +duckduckgo-search + +# Testing frameworks +pytest==7.4.4 +pytest-vcr==1.0.2 +pytest-asyncio==0.21.0 + +# General dependencies +pydantic==2.8.2 +httpx==0.27.2 +Deprecated==1.2.14 +importlib-metadata==6.11.0 +packaging==24.0 +wrapt==1.16.0 + +# Local development packages + +-e opentelemetry-instrumentation +-e instrumentation-genai/opentelemetry-instrumentation-langchain diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_agents.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_agents.py new file mode 100644 index 0000000000..9c484f9ed6 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_agents.py @@ -0,0 +1,137 @@ +import os +from typing import Tuple + +import pytest +from langchain import hub +from langchain_aws import ChatBedrock +from langchain.agents import AgentExecutor, create_tool_calling_agent +from langchain_community.tools import DuckDuckGoSearchResults + +@pytest.mark.vcr +def test_agents(instrument_legacy, span_exporter, log_exporter): + search = DuckDuckGoSearchResults() + tools = [search] + model = ChatBedrock( + model_id="anthropic.claude-3-5-sonnet-20240620-v1:0", + region_name="us-west-2", + temperature=0.9, + max_tokens=2048, + model_kwargs={ + "top_p": 0.9, + }, + ) + + prompt = hub.pull( + "hwchase17/openai-functions-agent", + api_key=os.environ["LANGSMITH_API_KEY"], + ) + + agent = create_tool_calling_agent(model, tools, prompt) + agent_executor = AgentExecutor(agent=agent, tools=tools) + + agent_executor.invoke({"input": "When was Amazon founded?"}) + + spans = span_exporter.get_finished_spans() + + assert set([span.name for span in spans]) == { + "chat anthropic.claude-3-5-sonnet-20240620-v1:0", + "chain LLMChain", + "chain AgentExecutor", + "execute_tool search", + "chain RunnableSequence", + "chain ToolsAgentOutputParser", + "chain ChatPromptTemplate", + "chain RunnableAssign", + "chain RunnableParallel", + "chain RunnableLambda", + "execute_tool duckduckgo_results_json", + } + + +@pytest.mark.vcr +def test_agents_with_events_with_content( + instrument_with_content, span_exporter, log_exporter +): + search = DuckDuckGoSearchResults() + tools = [search] + model = ChatBedrock( + model_id="anthropic.claude-3-5-sonnet-20240620-v1:0", + region_name="us-west-2", + temperature=0.9, + max_tokens=2048, + model_kwargs={ + "top_p": 0.9, + }, + ) + + + prompt = hub.pull( + "hwchase17/openai-functions-agent", + api_key=os.environ["LANGSMITH_API_KEY"], + ) + + agent = create_tool_calling_agent(model, tools, prompt) + agent_executor = AgentExecutor(agent=agent, tools=tools) + + + prompt = "What is AWS?" + response = agent_executor.invoke({"input": prompt}) + + spans = span_exporter.get_finished_spans() + + assert set([span.name for span in spans]) == { + "chat anthropic.claude-3-5-sonnet-20240620-v1:0", + "chain LLMChain", + "chain AgentExecutor", + "execute_tool search", + "chain RunnableSequence", + "chain ToolsAgentOutputParser", + "chain ChatPromptTemplate", + "chain RunnableAssign", + "chain RunnableParallel", + "chain RunnableLambda", + "execute_tool duckduckgo_results_json", + } + + +@pytest.mark.vcr +def test_agents_with_events_with_no_content( + instrument_with_no_content, span_exporter, log_exporter +): + search = DuckDuckGoSearchResults() + tools = [search] + model = ChatBedrock( + model_id="anthropic.claude-3-5-sonnet-20240620-v1:0", + region_name="us-west-2", + temperature=0.9, + max_tokens=2048, + model_kwargs={ + "top_p": 0.9, + }, + ) + + prompt = hub.pull( + "hwchase17/openai-functions-agent", + api_key=os.environ["LANGSMITH_API_KEY"], + ) + + agent = create_tool_calling_agent(model, tools, prompt) + agent_executor = AgentExecutor(agent=agent, tools=tools) + + agent_executor.invoke({"input": "What is AWS?"}) + + spans = span_exporter.get_finished_spans() + + assert set([span.name for span in spans]) == { + "chat anthropic.claude-3-5-sonnet-20240620-v1:0", + "chain LLMChain", + "chain AgentExecutor", + "execute_tool search", + "chain RunnableSequence", + "chain ToolsAgentOutputParser", + "chain ChatPromptTemplate", + "chain RunnableAssign", + "chain RunnableParallel", + "chain RunnableLambda", + "execute_tool duckduckgo_results_json", + } diff --git a/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_chains.py b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_chains.py new file mode 100644 index 0000000000..b588d5995f --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-langchain/tests/test_chains.py @@ -0,0 +1,115 @@ +import os +from typing import Tuple + +import pytest +from langchain import hub +from langchain_aws import ChatBedrock +from langchain.agents import AgentExecutor, create_tool_calling_agent +from langchain_community.tools import DuckDuckGoSearchResults + +import boto3 +from langchain_aws import BedrockLLM + +from langchain.chains import LLMChain, SequentialChain + +@pytest.mark.vcr +def test_sequential_chain(instrument_legacy, span_exporter, log_exporter): + bedrock_client = boto3.client( + service_name='bedrock-runtime', + region_name='us-west-2' + ) + + llm = BedrockLLM( + client=bedrock_client, + model_id="anthropic.claude-v2", + model_kwargs={ + "max_tokens_to_sample": 500, + "temperature": 0.7, + }, + ) + synopsis_template = """You are a playwright. Given the title of play and the era it is set in, it is your job to write a synopsis for that title. + + Title: {title} + Era: {era} + Playwright: This is a synopsis for the above play:""" # noqa: E501 + synopsis_prompt_template = PromptTemplate( + input_variables=["title", "era"], template=synopsis_template + ) + synopsis_chain = LLMChain( + llm=llm, prompt=synopsis_prompt_template, output_key="synopsis", name="synopsis" + ) + + template = """You are a play critic from the New York Times. Given the synopsis of play, it is your job to write a review for that play. + + Play Synopsis: + {synopsis} + Review from a New York Times play critic of the above play:""" # noqa: E501 + prompt_template = PromptTemplate(input_variables=["synopsis"], template=template) + review_chain = LLMChain(llm=llm, prompt=prompt_template, output_key="review") + + overall_chain = SequentialChain( + chains=[synopsis_chain, review_chain], + input_variables=["era", "title"], + # Here we return multiple variables + output_variables=["synopsis", "review"], + verbose=True, + ) + overall_chain.invoke( + {"title": "Tragedy at sunset on the beach", "era": "Victorian England"} + ) + + spans = span_exporter.get_finished_spans() + + langchain_spans = [ + span for span in spans + if span.name.startswith("chain ") + ] + + assert [ + "chain synopsis", + "chain LLMChain", + "chain SequentialChain", + ] == [span.name for span in langchain_spans] + + synopsis_span = next(span for span in spans if span.name == "chain synopsis") + review_span = next(span for span in spans if span.name == "chain LLMChain") + overall_span = next(span for span in spans if span.name == "chain SequentialChain") + + assert synopsis_span.kind == "SpanKind.INTERNAL" + assert "gen_ai.prompt" in synopsis_span.attributes + assert "gen_ai.completion" in synopsis_span.attributes + + synopsis_prompt = json.loads(synopsis_span.attributes["gen_ai.prompt"].replace("'", "\"")) + synopsis_completion = json.loads(synopsis_span.attributes["gen_ai.completion"].replace("'", "\"")) + + assert synopsis_prompt == { + "title": "Tragedy at sunset on the beach", + "era": "Victorian England" + } + assert "synopsis" in synopsis_completion + + assert review_span.kind == "SpanKind.INTERNAL" + assert "gen_ai.prompt" in review_span.attributes + assert "gen_ai.completion" in review_span.attributes + + review_prompt = json.loads(review_span.attributes["gen_ai.prompt"].replace("'", "\"")) + review_completion = json.loads(review_span.attributes["gen_ai.completion"].replace("'", "\"")) + + assert "title" in review_prompt + assert "era" in review_prompt + assert "synopsis" in review_prompt + assert "review" in review_completion + + assert overall_span.kind == "SpanKind.INTERNAL" + assert "gen_ai.prompt" in overall_span.attributes + assert "gen_ai.completion" in overall_span.attributes + + overall_prompt = json.loads(overall_span.attributes["gen_ai.prompt"].replace("'", "\"")) + overall_completion = json.loads(overall_span.attributes["gen_ai.completion"].replace("'", "\"")) + + assert overall_prompt == { + "title": "Tragedy at sunset on the beach", + "era": "Victorian England" + } + assert "synopsis" in overall_completion + assert "review" in overall_completion \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 6dbccdf541..4b92efefd7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,6 +66,7 @@ dependencies = [ "opentelemetry-util-http", "opentelemetry-instrumentation-vertexai[instruments]", "opentelemetry-instrumentation-openai-v2[instruments]", + "opentelemetry-instrumentation-langchain[instruments]", ] @@ -136,6 +137,7 @@ opentelemetry-propagator-aws-xray = { workspace = true } opentelemetry-util-http = { workspace = true } opentelemetry-instrumentation-vertexai = { workspace = true } opentelemetry-instrumentation-openai-v2 = { workspace = true } +opentelemetry-instrumentation-langchain = { workspace = true } # https://docs.astral.sh/uv/reference/settings/#workspace [tool.uv.workspace] diff --git a/tox.ini b/tox.ini index b37982567a..17bc2edfed 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,11 @@ envlist = ; Environments are organized by individual package, allowing ; for specifying supported Python versions per package. + ; instrumentation-openai + py3{9,10,11,12,13}-test-instrumentation-langchain-{oldest,latest} + pypy3-test-instrumentation-langchain-{oldest,latest} + lint-instrumentation-langchain + ; instrumentation-openai py3{9,10,11,12,13}-test-instrumentation-openai-v2-{oldest,latest} pypy3-test-instrumentation-openai-v2-{oldest,latest}