Skip to content

feat: handle LLOs in TracerProvider pipeline [WIP] #357

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 12 commits into from
Closed

feat: handle LLOs in TracerProvider pipeline [WIP] #357

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
e617f6d
set up core logic for llo data handling
yiyuan-he May 1, 2025
e87dc3d
add detection logic for llo regex patterns
yiyuan-he May 1, 2025
498a87f
add support for openinference llo string attributes
yiyuan-he May 1, 2025
68255a1
add support for llo attributes in span events
yiyuan-he May 2, 2025
28f493d
add sigv4 logs exporter poc
yiyuan-he May 2, 2025
c0f8a05
set up llo handling abstraction
yiyuan-he May 7, 2025
68f031e
set up agentic monitoring feature flag
yiyuan-he May 7, 2025
15832d0
emit llo to log pipeline
yiyuan-he May 7, 2025
b19a7ba
use dependency injection for logs exporter in agentic observability
yiyuan-he May 8, 2025
b5adb61
set up framework extensible extractions using reconstruction
yiyuan-he May 8, 2025
b30b053
simplify extraction logic and set up custom gen ai event for traceloop
yiyuan-he May 9, 2025
a1dcbaf
add original attribute to gen ai event
yiyuan-he May 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
110 changes: 55 additions & 55 deletions aws-opentelemetry-distro/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,61 +25,61 @@ classifiers = [
]

dependencies = [
"opentelemetry-api == 1.27.0",
"opentelemetry-sdk == 1.27.0",
"opentelemetry-exporter-otlp-proto-grpc == 1.27.0",
"opentelemetry-exporter-otlp-proto-http == 1.27.0",
"opentelemetry-propagator-b3 == 1.27.0",
"opentelemetry-propagator-jaeger == 1.27.0",
"opentelemetry-exporter-otlp-proto-common == 1.27.0",
"opentelemetry-sdk-extension-aws == 2.0.2",
"opentelemetry-propagator-aws-xray == 1.0.1",
"opentelemetry-distro == 0.48b0",
"opentelemetry-propagator-ot-trace == 0.48b0",
"opentelemetry-instrumentation == 0.48b0",
"opentelemetry-instrumentation-aws-lambda == 0.48b0",
"opentelemetry-instrumentation-aio-pika == 0.48b0",
"opentelemetry-instrumentation-aiohttp-client == 0.48b0",
"opentelemetry-instrumentation-aiopg == 0.48b0",
"opentelemetry-instrumentation-asgi == 0.48b0",
"opentelemetry-instrumentation-asyncpg == 0.48b0",
"opentelemetry-instrumentation-boto == 0.48b0",
"opentelemetry-instrumentation-boto3sqs == 0.48b0",
"opentelemetry-instrumentation-botocore == 0.48b0",
"opentelemetry-instrumentation-celery == 0.48b0",
"opentelemetry-instrumentation-confluent-kafka == 0.48b0",
"opentelemetry-instrumentation-dbapi == 0.48b0",
"opentelemetry-instrumentation-django == 0.48b0",
"opentelemetry-instrumentation-elasticsearch == 0.48b0",
"opentelemetry-instrumentation-falcon == 0.48b0",
"opentelemetry-instrumentation-fastapi == 0.48b0",
"opentelemetry-instrumentation-flask == 0.48b0",
"opentelemetry-instrumentation-grpc == 0.48b0",
"opentelemetry-instrumentation-httpx == 0.48b0",
"opentelemetry-instrumentation-jinja2 == 0.48b0",
"opentelemetry-instrumentation-kafka-python == 0.48b0",
"opentelemetry-instrumentation-logging == 0.48b0",
"opentelemetry-instrumentation-mysql == 0.48b0",
"opentelemetry-instrumentation-mysqlclient == 0.48b0",
"opentelemetry-instrumentation-pika == 0.48b0",
"opentelemetry-instrumentation-psycopg2 == 0.48b0",
"opentelemetry-instrumentation-pymemcache == 0.48b0",
"opentelemetry-instrumentation-pymongo == 0.48b0",
"opentelemetry-instrumentation-pymysql == 0.48b0",
"opentelemetry-instrumentation-pyramid == 0.48b0",
"opentelemetry-instrumentation-redis == 0.48b0",
"opentelemetry-instrumentation-remoulade == 0.48b0",
"opentelemetry-instrumentation-requests == 0.48b0",
"opentelemetry-instrumentation-sqlalchemy == 0.48b0",
"opentelemetry-instrumentation-sqlite3 == 0.48b0",
"opentelemetry-instrumentation-starlette == 0.48b0",
"opentelemetry-instrumentation-system-metrics == 0.48b0",
"opentelemetry-instrumentation-tornado == 0.48b0",
"opentelemetry-instrumentation-tortoiseorm == 0.48b0",
"opentelemetry-instrumentation-urllib == 0.48b0",
"opentelemetry-instrumentation-urllib3 == 0.48b0",
"opentelemetry-instrumentation-wsgi == 0.48b0",
"opentelemetry-instrumentation-cassandra == 0.48b0",
"opentelemetry-api >= 1.29.0",
"opentelemetry-sdk >= 1.29.0",
"opentelemetry-exporter-otlp-proto-grpc >= 1.29.0",
"opentelemetry-exporter-otlp-proto-http >= 1.29.0",
"opentelemetry-propagator-b3 >= 1.29.0",
"opentelemetry-propagator-jaeger >= 1.29.0",
"opentelemetry-exporter-otlp-proto-common >= 1.29.0",
"opentelemetry-sdk-extension-aws >= 2.0.2",
"opentelemetry-propagator-aws-xray >= 1.0.1",
"opentelemetry-distro >= 0.50b0",
"opentelemetry-propagator-ot-trace >= 0.50b0",
"opentelemetry-instrumentation >= 0.50b0",
"opentelemetry-instrumentation-aws-lambda >= 0.50b0",
"opentelemetry-instrumentation-aio-pika >= 0.50b0",
"opentelemetry-instrumentation-aiohttp-client >= 0.50b0",
"opentelemetry-instrumentation-aiopg >= 0.50b0",
"opentelemetry-instrumentation-asgi >= 0.50b0",
"opentelemetry-instrumentation-asyncpg >= 0.50b0",
"opentelemetry-instrumentation-boto >= 0.50b0",
"opentelemetry-instrumentation-boto3sqs >= 0.50b0",
"opentelemetry-instrumentation-botocore >= 0.50b0",
"opentelemetry-instrumentation-celery >= 0.50b0",
"opentelemetry-instrumentation-confluent-kafka >= 0.50b0",
"opentelemetry-instrumentation-dbapi >= 0.50b0",
"opentelemetry-instrumentation-django >= 0.50b0",
"opentelemetry-instrumentation-elasticsearch >= 0.50b0",
"opentelemetry-instrumentation-falcon >= 0.50b0",
"opentelemetry-instrumentation-fastapi >= 0.50b0",
"opentelemetry-instrumentation-flask >= 0.50b0",
"opentelemetry-instrumentation-grpc >= 0.50b0",
"opentelemetry-instrumentation-httpx >= 0.50b0",
"opentelemetry-instrumentation-jinja2 >= 0.50b0",
"opentelemetry-instrumentation-kafka-python >= 0.50b0",
"opentelemetry-instrumentation-logging >= 0.50b0",
"opentelemetry-instrumentation-mysql >= 0.50b0",
"opentelemetry-instrumentation-mysqlclient >= 0.50b0",
"opentelemetry-instrumentation-pika >= 0.50b0",
"opentelemetry-instrumentation-psycopg2 >= 0.50b0",
"opentelemetry-instrumentation-pymemcache >= 0.50b0",
"opentelemetry-instrumentation-pymongo >= 0.50b0",
"opentelemetry-instrumentation-pymysql >= 0.50b0",
"opentelemetry-instrumentation-pyramid >= 0.50b0",
"opentelemetry-instrumentation-redis >= 0.50b0",
"opentelemetry-instrumentation-remoulade >= 0.50b0",
"opentelemetry-instrumentation-requests >= 0.50b0",
"opentelemetry-instrumentation-sqlalchemy >= 0.50b0",
"opentelemetry-instrumentation-sqlite3 >= 0.50b0",
"opentelemetry-instrumentation-starlette >= 0.50b0",
"opentelemetry-instrumentation-system-metrics >= 0.50b0",
"opentelemetry-instrumentation-tornado >= 0.50b0",
"opentelemetry-instrumentation-tortoiseorm >= 0.50b0",
"opentelemetry-instrumentation-urllib >= 0.50b0",
"opentelemetry-instrumentation-urllib3 >= 0.50b0",
"opentelemetry-instrumentation-wsgi >= 0.50b0",
"opentelemetry-instrumentation-cassandra >= 0.50b0",
]

[project.optional-dependencies]
Expand Down
41 changes: 37 additions & 4 deletions aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
# Modifications Copyright The OpenTelemetry Authors. Licensed under the Apache License 2.0 License.
import os
import re
from logging import Logger, getLogger
from logging import NOTSET, Logger, getLogger
from typing import ClassVar, Dict, List, Type, Union

from importlib_metadata import version
Expand All @@ -21,11 +21,13 @@
AwsMetricAttributesSpanExporterBuilder,
)
from amazon.opentelemetry.distro.aws_span_metrics_processor_builder import AwsSpanMetricsProcessorBuilder
from amazon.opentelemetry.distro.otlp_aws_logs_exporter import OTLPAwsLogExporter
from amazon.opentelemetry.distro.otlp_aws_span_exporter import OTLPAwsSpanExporter
from amazon.opentelemetry.distro.otlp_udp_exporter import OTLPUdpSpanExporter
from amazon.opentelemetry.distro.sampler.aws_xray_remote_sampler import AwsXRayRemoteSampler
from amazon.opentelemetry.distro.scope_based_exporter import ScopeBasedPeriodicExportingMetricReader
from amazon.opentelemetry.distro.scope_based_filtering_view import ScopeBasedRetainingView
from opentelemetry._logs import set_logger_provider
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter as OTLPHttpOTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.metrics import set_meter_provider
Expand All @@ -36,9 +38,10 @@
_import_exporters,
_import_id_generator,
_import_sampler,
_init_logging,
_OTelSDKConfigurator,
)
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor, LogExporter
from opentelemetry.sdk.environment_variables import (
_OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED,
OTEL_EXPORTER_OTLP_METRICS_PROTOCOL,
Expand Down Expand Up @@ -84,6 +87,8 @@
OTEL_AWS_PYTHON_DEFER_TO_WORKERS_ENABLED_CONFIG = "OTEL_AWS_PYTHON_DEFER_TO_WORKERS_ENABLED"
SYSTEM_METRICS_INSTRUMENTATION_SCOPE_NAME = "opentelemetry.instrumentation.system_metrics"
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT = "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"
OTEL_EXPORTER_OTLP_LOGS_ENDPOINT = "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT"
AGENT_OBSERVABILITY_ENABLED = "true"
XRAY_OTLP_ENDPOINT_PATTERN = r"https://xray\.([a-z0-9-]+)\.amazonaws\.com/v1/traces$"
# UDP package size is not larger than 64KB
LAMBDA_SPAN_EXPORT_BATCH_SIZE = 10
Expand Down Expand Up @@ -160,6 +165,27 @@ def _initialize_components():
_init_logging(log_exporters, resource)


def _init_logging(
exporters: Dict[str, Type[LogExporter]],
resource: Resource = None,
):
provider = LoggerProvider(resource=resource)
set_logger_provider(provider)

for _, exporter_class in exporters.items():
exporter_args = {}
log_exporter = _customize_logs_exporter(exporter_class(**exporter_args), resource)
provider.add_log_record_processor(BatchLogRecordProcessor(exporter=log_exporter))

handler = LoggingHandler(level=NOTSET, logger_provider=provider)

getLogger().addHandler(handler)


def _customize_logs_exporter(log_exporter: LogExporter, resource: Resource) -> LogExporter:
return OTLPAwsLogExporter(endpoint=os.getenv(OTEL_EXPORTER_OTLP_LOGS_ENDPOINT))
Copy link
Member

@mxiamxia mxiamxia May 6, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what will be the default logs_endpoint if it is not specified by Env var?

Copy link
Contributor Author

@yiyuan-he yiyuan-he May 7, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I will align with the default from the SigV4 Logs Exporter PR. I plan to drop my PoC implementation and plug that in once it is merged.



def _init_tracing(
exporters: Dict[str, Type[SpanExporter]],
id_generator: IdGenerator = None,
Expand Down Expand Up @@ -324,8 +350,15 @@ def _customize_exporter(span_exporter: SpanExporter, resource: Resource) -> Span
_logger.info("Detected using AWS OTLP XRay Endpoint.")

if isinstance(span_exporter, OTLPSpanExporter):
span_exporter = OTLPAwsSpanExporter(endpoint=os.getenv(OTEL_EXPORTER_OTLP_TRACES_ENDPOINT))

if AGENT_OBSERVABILITY_ENABLED == "true":
logs_endpoint = os.getenv(OTEL_EXPORTER_OTLP_LOGS_ENDPOINT)
logs_exporter = OTLPAwsLogExporter(endpoint=logs_endpoint)
span_exporter = OTLPAwsSpanExporter(
endpoint=os.getenv(OTEL_EXPORTER_OTLP_TRACES_ENDPOINT),
logs_exporter=logs_exporter
)
else:
span_exporter = OTLPAwsSpanExporter(endpoint=os.getenv(OTEL_EXPORTER_OTLP_TRACES_ENDPOINT))
else:
_logger.warning(
"Improper configuration see: please export/set "
Expand Down
Loading