Feature/tl translator evals admehra

instrumentation-genai/opentelemetry-instrumentation-langchain-dev/examples/multi_agent_travel_planner/.env

@@ -0,0 +1,8 @@
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
+OTEL_SEMCONV_STABILITY_OPT_IN=gen_ai_latest_experimental
+OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=true
+OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT_MODE=SPAN_AND_EVENT
+OTEL_INSTRUMENTATION_GENAI_EMITTERS=span_metric_event,splunk
+OTEL_INSTRUMENTATION_GENAI_EMITTERS_EVALUATION=replace-category:SplunkEvaluationResults
+OTEL_INSTRUMENTATION_GENAI_EVALS_RESULTS_AGGREGATION=true
+OTEL_INSTRUMENTATION_GENAI_DEBUG=true

instrumentation-genai/opentelemetry-instrumentation-langchain-dev/examples/multi_agent_travel_planner/requirements.admehra.txt

@@ -0,0 +1,155 @@
+aiohappyeyeballs==2.6.1
+aiohttp==3.13.2
+aiosignal==1.4.0
+annotated-types==0.7.0
+anthropic==0.72.0
+anyio==4.11.0
+attrs==25.4.0
+backoff==2.2.1
+cachetools==6.2.1
+certifi==2025.10.5
+charset-normalizer==3.4.4
+click==8.2.1
+colorama==0.4.6
+cuid==0.4
+deepeval==3.3.9
+Deprecated==1.3.1
+distro==1.9.0
+docstring_parser==0.17.0
+execnet==2.1.1
+filelock==3.20.0
+frozenlist==1.8.0
+fsspec==2025.10.0
+google-auth==2.42.1
+google-genai==1.47.0
+googleapis-common-protos==1.71.0
+grpcio==1.76.0
+h11==0.16.0
+hf-xet==1.2.0
+httpcore==1.0.9
+httpx==0.28.1
+huggingface-hub==1.0.1
+idna==3.11
+importlib_metadata==8.7.0
+inflection==0.5.1
+iniconfig==2.3.0
+Jinja2==3.1.6
+jiter==0.11.1
+jsonpatch==1.33
+jsonpointer==3.0.0
+langchain==1.0.3
+langchain-core==1.0.2
+langchain-openai==1.0.1
+langgraph==1.0.2
+langgraph-checkpoint==3.0.0
+langgraph-prebuilt==1.0.2
+langgraph-sdk==0.2.9
+langsmith==0.4.39
+markdown-it-py==4.0.0
+MarkupSafe==3.0.3
+mdurl==0.1.2
+monotonic==1.6
+multidict==6.7.0
+nest-asyncio==1.6.0
+ollama==0.6.0
+openai==2.6.1
+opentelemetry-api==1.38.0
+opentelemetry-exporter-otlp==1.38.0
+opentelemetry-exporter-otlp-proto-common==1.38.0
+opentelemetry-exporter-otlp-proto-grpc==1.38.0
+opentelemetry-exporter-otlp-proto-http==1.38.0
+opentelemetry-instrumentation==0.59b0
+opentelemetry-instrumentation-alephalpha==0.47.5
+opentelemetry-instrumentation-anthropic==0.47.5
+opentelemetry-instrumentation-bedrock==0.47.5
+opentelemetry-instrumentation-chromadb==0.47.5
+opentelemetry-instrumentation-cohere==0.47.5
+opentelemetry-instrumentation-crewai==0.47.5
+opentelemetry-instrumentation-google-generativeai==0.47.5
+opentelemetry-instrumentation-groq==0.47.5
+opentelemetry-instrumentation-haystack==0.47.5
+opentelemetry-instrumentation-lancedb==0.47.5
+-e git+ssh://[email protected]/zhirafovod/opentelemetry-python-contrib.git@7b13c32a3b1c8ad66bee8c49e31120cc586fa1ad#egg=opentelemetry_instrumentation_langchain&subdirectory=instrumentation-genai/opentelemetry-instrumentation-langchain-dev
+opentelemetry-instrumentation-llamaindex==0.47.5
+opentelemetry-instrumentation-logging==0.59b0
+opentelemetry-instrumentation-marqo==0.47.5
+opentelemetry-instrumentation-mcp==0.47.5
+opentelemetry-instrumentation-milvus==0.47.5
+opentelemetry-instrumentation-mistralai==0.47.5
+opentelemetry-instrumentation-ollama==0.47.5
+opentelemetry-instrumentation-openai==0.47.5
+opentelemetry-instrumentation-openai-agents==0.47.5
+opentelemetry-instrumentation-pinecone==0.47.5
+opentelemetry-instrumentation-qdrant==0.47.5
+opentelemetry-instrumentation-redis==0.59b0
+opentelemetry-instrumentation-replicate==0.47.5
+opentelemetry-instrumentation-requests==0.59b0
+opentelemetry-instrumentation-sagemaker==0.47.5
+opentelemetry-instrumentation-sqlalchemy==0.59b0
+opentelemetry-instrumentation-threading==0.59b0
+opentelemetry-instrumentation-together==0.47.5
+opentelemetry-instrumentation-transformers==0.47.5
+opentelemetry-instrumentation-urllib3==0.59b0
+opentelemetry-instrumentation-vertexai==0.47.5
+opentelemetry-instrumentation-watsonx==0.47.5
+opentelemetry-instrumentation-weaviate==0.47.5
+opentelemetry-instrumentation-writer==0.47.5
+opentelemetry-proto==1.38.0
+opentelemetry-sdk==1.38.0
+opentelemetry-semantic-conventions==0.59b0
+opentelemetry-semantic-conventions-ai==0.4.13
+-e git+ssh://[email protected]/zhirafovod/opentelemetry-python-contrib.git@7b13c32a3b1c8ad66bee8c49e31120cc586fa1ad#egg=opentelemetry_util_genai&subdirectory=util/opentelemetry-util-genai-dev
+-e git+ssh://[email protected]/zhirafovod/opentelemetry-python-contrib.git@7b13c32a3b1c8ad66bee8c49e31120cc586fa1ad#egg=opentelemetry_util_genai_evals&subdirectory=util/opentelemetry-util-genai-evals
+-e git+ssh://[email protected]/zhirafovod/opentelemetry-python-contrib.git@7b13c32a3b1c8ad66bee8c49e31120cc586fa1ad#egg=opentelemetry_util_genai_evals_deepeval&subdirectory=util/opentelemetry-util-genai-evals-deepeval
+-e git+ssh://[email protected]/zhirafovod/opentelemetry-python-contrib.git@7b13c32a3b1c8ad66bee8c49e31120cc586fa1ad#egg=opentelemetry_util_genai_traceloop_translator&subdirectory=util/opentelemetry-util-genai-traceloop-translator
+-e git+ssh://[email protected]/zhirafovod/opentelemetry-python-contrib.git@7b13c32a3b1c8ad66bee8c49e31120cc586fa1ad#egg=opentelemetry_util_http&subdirectory=util/opentelemetry-util-http
+orjson==3.11.4
+ormsgpack==1.11.0
+packaging==25.0
+pluggy==1.6.0
+portalocker==3.2.0
+posthog==3.25.0
+propcache==0.4.1
+protobuf==6.33.0
+pyasn1==0.6.1
+pyasn1_modules==0.4.2
+pydantic==2.12.3
+pydantic_core==2.41.4
+pyfiglet==1.0.4
+Pygments==2.19.2
+pytest==8.4.2
+pytest-asyncio==1.2.0
+pytest-repeat==0.9.4
+pytest-rerunfailures==12.0
+pytest-xdist==3.8.0
+python-dateutil==2.9.0.post0
+python-dotenv==1.2.1
+PyYAML==6.0.3
+regex==2025.10.23
+requests==2.32.5
+requests-toolbelt==1.0.0
+rich==14.2.0
+rsa==4.9.1
+sentry-sdk==2.43.0
+setuptools==80.9.0
+shellingham==1.5.4
+six==1.17.0
+sniffio==1.3.1
+tabulate==0.9.0
+tenacity==9.1.2
+tiktoken==0.12.0
+tokenizers==0.22.1
+tqdm==4.67.1
+traceloop-sdk==0.47.5
+typer==0.20.0
+typer-slim==0.20.0
+typing-inspection==0.4.2
+typing_extensions==4.15.0
+urllib3==2.5.0
+websockets==15.0.1
+wheel==0.45.1
+wrapt==1.17.3
+xxhash==3.6.0
+yarl==1.22.0
+zipp==3.23.0
+zstandard==0.25.0

util/opentelemetry-util-genai-dev/src/opentelemetry/util/genai/emitters/span.py

@@ -65,6 +65,26 @@
 _SPAN_ALLOWED_SUPPLEMENTAL_KEYS: tuple[str, ...] = (
     "gen_ai.framework",
     "gen_ai.request.id",
+    # Workflow / Agent attributes (from Traceloop translator)
+    "gen_ai.workflow.name",
+    "gen_ai.workflow.path",
+    "gen_ai.workflow.version",
+    "gen_ai.agent.name",
+    "gen_ai.agent.id",
+    "gen_ai.span.kind",
+    "gen_ai.association.properties",
+    "gen_ai.conversation.id",
+    # Prompt registry attributes
+    "gen_ai.prompt.managed",
+    "gen_ai.prompt.key",
+    "gen_ai.prompt.version",
+    "gen_ai.prompt.version_name",
+    "gen_ai.prompt.version_hash",
+    "gen_ai.prompt.template",
+    "gen_ai.prompt.template_variables",
+    # Operation name override (for span name transformation)
+    "gen_ai.override.span_name",
+    # Custom attributes (allow any custom.* prefix for flexibility)
 )
 _SPAN_BLOCKED_SUPPLEMENTAL_KEYS: set[str] = {"request_top_p", "ls_temperature"}
 
@@ -297,16 +317,25 @@ def on_start(
         elif isinstance(invocation, EmbeddingInvocation):
             self._start_embedding(invocation)
         else:
-            # Use operation field for span name (defaults to "chat")
-            operation = getattr(invocation, "operation", "chat")
-            model_name = invocation.request_model
-            span_name = f"{operation} {model_name}"
-            parent_span = getattr(invocation, "parent_span", None)
-            parent_ctx = (
-                trace.set_span_in_context(parent_span)
-                if parent_span is not None
-                else None
-            )
+            # Check for span name override first (for Traceloop task/workflow spans)
+            span_name_override = invocation.attributes.get("gen_ai.override.span_name")
+            if span_name_override:
+                span_name = str(span_name_override)
+            else:
+                # Use operation field for span name (defaults to "chat")
+                operation = getattr(invocation, "operation", "chat")
+                model_name = invocation.request_model
+                span_name = f"{operation} {model_name}"
+
+            # Check for parent context (from TraceloopSpanProcessor) or parent span
+            parent_ctx = getattr(invocation, "parent_context", None)
+            if parent_ctx is None:
+                parent_span = getattr(invocation, "parent_span", None)
+                parent_ctx = (
+                    trace.set_span_in_context(parent_span)
+                    if parent_span is not None
+                    else None
+                )
             cm = self._tracer.start_as_current_span(
                 span_name,
                 kind=SpanKind.CLIENT,

util/opentelemetry-util-genai-dev/src/opentelemetry/util/genai/emitters/utils.py

@@ -128,9 +128,18 @@ def filter_semconv_gen_ai_attributes(
     for key, value in attributes.items():
         if not isinstance(key, str):
             continue
-        if key not in allowed:
+        # Allow exact matches
+        if key in allowed:
+            filtered[key] = value
             continue
-        filtered[key] = value
+        # Allow gen_ai.* attributes that follow semantic convention patterns
+        if key.startswith("gen_ai."):
+            # Check if it matches known patterns with array indices
+            # e.g., gen_ai.prompt.N.*, gen_ai.completion.N.*, gen_ai.choice.N.*
+            import re
+            if re.match(r"gen_ai\.(prompt|completion|choice)\.\d+\.", key):
+                filtered[key] = value
+                continue
     return filtered
 
 

util/opentelemetry-util-genai-dev/src/opentelemetry/util/genai/handler.py

@@ -69,6 +69,7 @@ def genai_debug_log(*_args: Any, **_kwargs: Any) -> None:  # type: ignore
 from opentelemetry import _logs
 from opentelemetry import metrics as _metrics
 from opentelemetry import trace as _trace_mod
+from opentelemetry.context import Context
 from opentelemetry.semconv.schemas import Schemas
 from opentelemetry.trace import get_tracer
 from opentelemetry.util.genai.emitters.configuration import (
@@ -341,8 +342,15 @@ def _refresh_capture_content(
     def start_llm(
         self,
         invocation: LLMInvocation,
+        parent_context: Context | None = None,
     ) -> LLMInvocation:
-        """Start an LLM invocation and create a pending span entry."""
+        """Start an LLM invocation and create a pending span entry.
+
+        Args:
+            invocation: The LLM invocation to start
+            parent_context: Optional parent context for the span. If provided, the new span
+                          will be a child of the span in this context.
+        """
         # Ensure capture content settings are current
         self._refresh_capture_content()
         genai_debug_log("handler.start_llm.begin", invocation)
@@ -355,6 +363,9 @@ def start_llm(
                 invocation.agent_name = top_name
             if not invocation.agent_id:
                 invocation.agent_id = top_id
+        # Store parent context if provided for emitter to use
+        if parent_context is not None:
+            invocation.parent_context = parent_context  # type: ignore[attr-defined]
         # Start invocation span; tracer context propagation handles parent/child links
         self._emitter.on_start(invocation)
         # Register span if created

util/opentelemetry-util-genai-dev/src/opentelemetry/util/genai/types.py

@@ -59,7 +59,42 @@ def _new_str_any_dict() -> dict[str, Any]:
 
 @dataclass(kw_only=True)
 class GenAI:
-    """Base type for all GenAI telemetry entities."""
+    """Base type for all GenAI telemetry entities.
+
+    All GenAI types (LLMInvocation, EmbeddingInvocation, AgentCreation, AgentInvocation,
+    Workflow, Step, ToolCall, etc.) inherit from this base class and have access to:
+
+    Span and Context Management:
+        - span: The OpenTelemetry span associated with this entity
+        - span_context: The span context for distributed tracing
+        - context_token: Token for managing span context lifecycle
+        - trace_id, span_id, trace_flags: Raw trace/span identifiers
+
+    Timing:
+        - start_time: When the operation started (auto-generated)
+        - end_time: When the operation completed
+
+    Identification:
+        - run_id: Unique identifier for this execution (auto-generated UUID)
+        - parent_run_id: Optional parent execution identifier for hierarchies
+
+    Provider and Framework:
+        - provider: AI provider (e.g., "openai", "anthropic", "cohere")
+        - framework: Framework used (e.g., "langchain", "llamaindex")
+        - system: AI system identifier (semantic convention attribute)
+
+    Agent and Conversation Context:
+        - agent_name, agent_id: Agent identification
+        - conversation_id: Conversation/session identifier
+        - data_source_id: Data source identifier for RAG scenarios
+
+    Behavior Control:
+        - mutate_original_span: Whether to modify the original span with transformations
+          (used by span processors like TraceloopSpanProcessor)
+
+    Custom Attributes:
+        - attributes: Dictionary for custom/additional attributes not covered by fields
+    """
 
     context_token: Optional[ContextToken] = None
     span: Optional[Span] = None
@@ -97,6 +132,14 @@ class GenAI:
         default=None,
         metadata={"semconv": GenAIAttributes.GEN_AI_DATA_SOURCE_ID},
     )
+    mutate_original_span: bool = field(
+        default=True,
+        metadata={
+            "description": "Whether to mutate the original span with transformed attributes. "
+            "When True, the original span will be modified with renamed/transformed attributes. "
+            "When False, only new spans will be created with the transformations."
+        },
+    )
 
     def semantic_convention_attributes(self) -> dict[str, Any]:
         """Return semantic convention attributes defined on this dataclass."""

util/opentelemetry-util-genai-traceloop-translator/.deepeval/.deepeval_telemetry.txt

@@ -0,0 +1,4 @@
+DEEPEVAL_ID=9d4289a3-58c6-41fe-869d-dc6a00930bea
+DEEPEVAL_STATUS=old
+DEEPEVAL_LAST_FEATURE=evaluation
+DEEPEVAL_EVALUATION_STATUS=old

util/opentelemetry-util-genai-traceloop-translator/.gitignore

@@ -0,0 +1,8 @@
+.env
+
+__pycache__/
+.vscode/
+*.pyc
+.DS_Store
+
+# Ignore example output files