New Gen-AI utils APIs creating semantic conventions compatible telemetry (POC)

instrumentation-genai/README.md

@@ -1,6 +1,7 @@
 
-| Instrumentation | Supported Packages | Metrics support | Semconv status |
-| --------------- | ------------------ | --------------- | -------------- |
-| [opentelemetry-instrumentation-google-genai](./opentelemetry-instrumentation-google-genai) | google-genai >= 1.0.0 | No | development
-| [opentelemetry-instrumentation-openai-v2](./opentelemetry-instrumentation-openai-v2) | openai >= 1.26.0 | Yes | development
-| [opentelemetry-instrumentation-vertexai](./opentelemetry-instrumentation-vertexai) | google-cloud-aiplatform >= 1.64 | No | development
+| Instrumentation                                                                            | Supported Packages              | Metrics support | Semconv status |
+|--------------------------------------------------------------------------------------------|---------------------------------|-----------------| -------------- |
+| [opentelemetry-instrumentation-google-genai](./opentelemetry-instrumentation-google-genai) | google-genai >= 1.0.0           | No              | development
+| [opentelemetry-instrumentation-openai-v2](./opentelemetry-instrumentation-openai-v2)       | openai >= 1.26.0                | Yes             | development
+| [opentelemetry-instrumentation-vertexai](./opentelemetry-instrumentation-vertexai)         | google-cloud-aiplatform >= 1.64 | No              | development
+| [opentelemetry-instrumentation-langchain](./opentelemetry-instrumentation-langchain)       | langchain >= 0.3.21               | Yes             | development

instrumentation-genai/opentelemetry-genai-sdk/README.rst

@@ -0,0 +1,27 @@
+Installation
+============
+
+Option 1: pip + requirements.txt
+---------------------------------
+::
+
+    python3 -m venv .venv
+    source .venv/bin/activate
+    pip install -r requirements.txt
+
+Option 2: Poetry
+----------------
+::
+
+    poetry install
+
+Running Tests
+=============
+
+After installing dependencies, simply run:
+
+::
+
+    pytest
+
+This will discover and run `tests/test_sdk.py`.

instrumentation-genai/opentelemetry-genai-sdk/pyproject.toml

@@ -0,0 +1,55 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "opentelemetry-genai-sdk"
+dynamic = ["version"]
+description = "OpenTelemetry GenAI SDK"
+readme = "README.rst"
+license = "Apache-2.0"
+requires-python = ">=3.8"
+authors = [
+  { name = "OpenTelemetry Authors", email = "[email protected]" },
+]
+classifiers = [
+  "Development Status :: 4 - Beta",
+  "Intended Audience :: Developers",
+  "License :: OSI Approved :: Apache Software License",
+  "Programming Language :: Python",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+]
+dependencies = [
+  "opentelemetry-api ~= 1.30",
+  "opentelemetry-instrumentation ~= 0.51b0",
+  "opentelemetry-semantic-conventions ~= 0.51b0",
+  "opentelemetry-api>=1.31.0",
+  "opentelemetry-sdk>=1.31.0",
+]
+
+[project.optional-dependencies]
+test = [
+  "pytest>=7.0.0",
+]
+# evaluation = ["deepevals>=0.1.0", "openlit-sdk>=0.1.0"]
+
+[project.urls]
+Homepage = "https://github.com/open-telemetry/opentelemetry-python-contrib/tree/main/instrumentation-genai/opentelemetry-genai-sdk"
+Repository = "https://github.com/open-telemetry/opentelemetry-python-contrib"
+
+[tool.hatch.version]
+path = "src/opentelemetry/genai/sdk/version.py"
+
+[tool.hatch.build.targets.sdist]
+include = [
+  "/src",
+  "/tests",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/opentelemetry"]

instrumentation-genai/opentelemetry-genai-sdk/requirements.txt

@@ -0,0 +1,10 @@
+# OpenTelemetry SDK
+opentelemetry-api>=1.34.0
+opentelemetry-sdk>=1.34.0
+
+# Testing
+pytest>=7.0.0
+
+# (Optional) evaluation libraries
+# deepevals>=0.1.0
+# openlit-sdk>=0.1.0

instrumentation-genai/opentelemetry-genai-sdk/src/opentelemetry/genai/sdk/api.py

@@ -0,0 +1,101 @@
+# Copyright The OpenTelemetry Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+from threading import Lock
+from typing import List, Optional
+from uuid import UUID
+
+from .types import LLMInvocation
+from .exporters import SpanMetricEventExporter, SpanMetricExporter
+from .data import Message, ChatGeneration, Error
+
+from opentelemetry.instrumentation.langchain.version import __version__
+from opentelemetry.metrics import get_meter
+from opentelemetry.trace import get_tracer
+from opentelemetry._events import get_event_logger
+from opentelemetry.semconv.schemas import Schemas
+
+
+class TelemetryClient:
+    """
+    High-level client managing GenAI invocation lifecycles and exporting
+    them as spans, metrics, and events.
+    """
+    def __init__(self, exporter_type_full: bool = True, **kwargs):
+        tracer_provider = kwargs.get("tracer_provider")
+        self._tracer = get_tracer(
+            __name__, __version__, tracer_provider, schema_url=Schemas.V1_28_0.value
+        )
+
+        meter_provider = kwargs.get("meter_provider")
+        self._meter = get_meter(
+            __name__, __version__, meter_provider, schema_url=Schemas.V1_28_0.value
+        )
+
+        event_logger_provider = kwargs.get("event_logger_provider")
+        self._event_logger = get_event_logger(
+            __name__, __version__, event_logger_provider=event_logger_provider, schema_url=Schemas.V1_28_0.value
+        )
+
+        self._exporter = (
+            SpanMetricEventExporter(tracer=self._tracer, meter=self._meter, event_logger=self._event_logger)
+            if exporter_type_full
+            else SpanMetricExporter(tracer=self._tracer, meter=self._meter)
+        )
+
+        self._llm_registry: dict[UUID, LLMInvocation] = {}
+        self._lock = Lock()
+
+    def start_llm(self, prompts: List[Message], run_id: UUID, parent_run_id: Optional[UUID] = None, **attributes):
+        invocation = LLMInvocation(messages=prompts , run_id=run_id, parent_run_id=parent_run_id, attributes=attributes)
+        with self._lock:
+            self._llm_registry[invocation.run_id] = invocation
+        self._exporter.init(invocation)
+
+    def stop_llm(self, run_id: UUID, chat_generations: List[ChatGeneration], **attributes) -> LLMInvocation:
+        with self._lock:
+            invocation = self._llm_registry.pop(run_id)
+        invocation.end_time = time.time()
+        invocation.chat_generations = chat_generations
+        invocation.attributes.update(attributes)
+        self._exporter.export(invocation)
+        return invocation
+
+    def fail_llm(self, run_id: UUID, error: Error, **attributes) -> LLMInvocation:
+        with self._lock:
+            invocation = self._llm_registry.pop(run_id)
+        invocation.end_time = time.time()
+        invocation.attributes.update(**attributes)
+        self._exporter.error(error, invocation)
+        return invocation
+
+# Singleton accessor
+_default_client: TelemetryClient | None = None
+
+def get_telemetry_client(exporter_type_full: bool = True, **kwargs) -> TelemetryClient:
+    global _default_client
+    if _default_client is None:
+        _default_client = TelemetryClient(exporter_type_full=exporter_type_full, **kwargs)
+    return _default_client
+
+# Module‐level convenience functions
+def llm_start(prompts: List[Message], run_id: UUID, parent_run_id: Optional[UUID] = None, **attributes):
+    return get_telemetry_client().start_llm(prompts=prompts, run_id=run_id, parent_run_id=parent_run_id, **attributes)
+
+def llm_stop(run_id: UUID, chat_generations: List[ChatGeneration], **attributes) -> LLMInvocation:
+    return get_telemetry_client().stop_llm(run_id=run_id, chat_generations=chat_generations, **attributes)
+
+def llm_fail(run_id: UUID, error: Error, **attributes) -> LLMInvocation:
+    return get_telemetry_client().fail_llm(run_id=run_id, error=error, **attributes)

instrumentation-genai/opentelemetry-genai-sdk/src/opentelemetry/genai/sdk/data.py

@@ -0,0 +1,18 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class Message:
+    content: str
+    type: str
+
+@dataclass
+class ChatGeneration:
+    content: str
+    type: str
+    finish_reason: str = None
+
+@dataclass
+class Error:
+    message: str
+    type: type[BaseException]

instrumentation-genai/opentelemetry-genai-sdk/src/opentelemetry/genai/sdk/evals.py

@@ -0,0 +1,69 @@
+from abc import ABC, abstractmethod
+from .types import LLMInvocation
+
+
+class EvaluationResult:
+    """
+    Standardized result for any GenAI evaluation.
+    """
+    def __init__(self, score: float, details: dict = None):
+        self.score = score
+        self.details = details or {}
+
+
+class Evaluator(ABC):
+    """
+    Abstract base: any evaluation backend must implement.
+    """
+    @abstractmethod
+    def evaluate(self, invocation: LLMInvocation) -> EvaluationResult:
+        """
+        Evaluate a completed LLMInvocation and return a result.
+        """
+        pass
+
+class DeepEvalsEvaluator(Evaluator):
+    """
+    Uses DeepEvals library for LLM-as-judge evaluations.
+    """
+    def __init__(self, config: dict = None):
+        # e.g. load models, setup API keys
+        self.config = config or {}
+
+    def evaluate(self, invocation: LLMInvocation) -> EvaluationResult:
+        # stub: integrate with deepevals SDK
+        # result = deepevals.judge(invocation.prompt, invocation.response, **self.config)
+        score = 0.0  # placeholder
+        details = {"method": "deepevals"}
+        return EvaluationResult(score=score, details=details)
+
+
+class OpenLitEvaluator(Evaluator):
+    """
+    Uses OpenLit or similar OSS evaluation library.
+    """
+    def __init__(self, config: dict = None):
+        self.config = config or {}
+
+    def evaluate(self, invocation: LLMInvocation) -> EvaluationResult:
+        # stub: integrate with openlit SDK
+        score = 0.0  # placeholder
+        details = {"method": "openlit"}
+        return EvaluationResult(score=score, details=details)
+
+
+# Registry for easy lookup
+EVALUATORS = {
+    "deepevals": DeepEvalsEvaluator,
+    "openlit": OpenLitEvaluator,
+}
+
+
+def get_evaluator(name: str, config: dict = None) -> Evaluator:
+    """
+    Factory: return an evaluator by name.
+    """
+    cls = EVALUATORS.get(name.lower())
+    if not cls:
+        raise ValueError(f"Unknown evaluator: {name}")
+    return cls(config)