Skip to content

Commit cb3a1b0

Browse files
authored
SigV4 Authentication Support for OTLP HTTP Logs Exporter (#358)
**Background** Supporting ADOT auto instrumentation to automatically inject SigV4 authentication headers for outgoing export log requests to the allow exporting to the AWS Logs OTLP endpoint. Users will need to configure the following environment variables in order to enable and properly run this exporter: `OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=https://logs.[AWS-REGION].amazonaws.com/v1/logs`; **required** `OTEL_EXPORTER_OTLP_LOGS_HEADERS`=`x-aws-log-group=[CW-LOG-GROUP-NAME],x-aws-log-stream=[CW-LOG-STREAM-NAME]` **required** `OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED`=`true` **required** `OTEL_LOGS_EXPORTER=otlp` **required or do not set env variable** `OTEL_EXPORTER_OTLP_LOGS_PROTOCOL=http/protobuf` **required or do not set env variable** `OTEL_METRICS_EXPORTER=none` **Description of changes:** 1. Add new AwsAuthSession class to inject Sigv4 headers directly into the sessions object used by the upstream exporter. https://github.com/srprash/aws-otel-sigv4-auth/tree/main 2. In order to ensure we don't override any user configurations from environment variables, the `OTLPAwsLogExporter` constructor copies all existing `LogsExporter` configurations create by upstream's instrumentation. 3. The ADOT auto instrumentation is now configured to automatically detect if a user is exporting to CW Logs OTLP Logs endpoint by checking if the environment variable `OTEL_EXPORTER_OTLP_LOGS_ENDPOINT` is configured to match this url pattern: `https://logs.[AWS-REGION].amazonaws.com/v1/logs` **Testing:** 1. E2E test done in an empty EC2 environment without configuring .aws credentials config file or setting AWS credentials in the environment variable 2. Manual testing was done by configuring the above environment variables and setting up the sample app locally with ADOT auto instrumentation and verified the spans in CW Logs. 3. The sample app was run and rerun 30 times and confirmed no issues with exporting the logs to the endpoint 4. Unit tests were added to verify functionality of OtlpAwsLogsExporter Further testing will be done with the Release tests. Example of a log exported using this exporter: ``` { "resource": { "attributes": { "aws.local.service": "test", "service.name": "test", "cloud.region": "us-west-2", "host.type": "c5.4xlarge", "cloud.availability_zone": "us-west-2c", "telemetry.sdk.name": "opentelemetry", "telemetry.sdk.language": "python", "cloud.provider": "aws", "cloud.account.id": "571600841604", "telemetry.sdk.version": "1.27.0", "host.name": "ip-172-31-7-29.us-west-2.compute.internal", "cloud.platform": "aws_ec2", "host.id": "i-0b04d6affbae7d629", "telemetry.auto.version": "0.9.0.dev0-aws" } }, "scope": { "name": "opentelemetry.sdk._logs._internal" }, "timeUnixNano": 1746221090210187520, "observedTimeUnixNano": 1746221090210645309, "severityNumber": 9, "severityText": "INFO", "body": "\u001b[31m\u001b[1mWARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead.\u001b[0m\n * Running on http://127.0.0.1:8082", "attributes": { "code.filepath": "/home/ec2-user/aws-otel-python-instrumentation/auto_instrumentation/lib64/python3.11/site-packages/werkzeug/_internal.py", "otelTraceSampled": false, "code.function": "_log", "code.lineno": 97, "otelTraceID": "0", "otelSpanID": "0", "otelServiceName": "test" }, "traceId": "", "spanId": "" } ```
1 parent db06fde commit cb3a1b0

File tree

6 files changed

+529
-330
lines changed

6 files changed

+529
-330
lines changed

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py

Lines changed: 112 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
# Modifications Copyright The OpenTelemetry Authors. Licensed under the Apache License 2.0 License.
44
import os
55
import re
6-
from logging import Logger, getLogger
6+
from logging import NOTSET, Logger, getLogger
77
from typing import ClassVar, Dict, List, Type, Union
88

99
from importlib_metadata import version
@@ -21,11 +21,14 @@
2121
AwsMetricAttributesSpanExporterBuilder,
2222
)
2323
from amazon.opentelemetry.distro.aws_span_metrics_processor_builder import AwsSpanMetricsProcessorBuilder
24-
from amazon.opentelemetry.distro.otlp_aws_span_exporter import OTLPAwsSpanExporter
24+
from amazon.opentelemetry.distro.exporter.otlp.aws.common.aws_auth_session import AwsAuthSession
2525
from amazon.opentelemetry.distro.otlp_udp_exporter import OTLPUdpSpanExporter
2626
from amazon.opentelemetry.distro.sampler.aws_xray_remote_sampler import AwsXRayRemoteSampler
2727
from amazon.opentelemetry.distro.scope_based_exporter import ScopeBasedPeriodicExportingMetricReader
2828
from amazon.opentelemetry.distro.scope_based_filtering_view import ScopeBasedRetainingView
29+
from opentelemetry._logs import set_logger_provider
30+
from opentelemetry.exporter.otlp.proto.http import Compression
31+
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
2932
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter as OTLPHttpOTLPMetricExporter
3033
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
3134
from opentelemetry.metrics import set_meter_provider
@@ -36,9 +39,10 @@
3639
_import_exporters,
3740
_import_id_generator,
3841
_import_sampler,
39-
_init_logging,
4042
_OTelSDKConfigurator,
4143
)
44+
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
45+
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor, LogExporter
4246
from opentelemetry.sdk.environment_variables import (
4347
_OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED,
4448
OTEL_EXPORTER_OTLP_METRICS_PROTOCOL,
@@ -84,7 +88,15 @@
8488
OTEL_AWS_PYTHON_DEFER_TO_WORKERS_ENABLED_CONFIG = "OTEL_AWS_PYTHON_DEFER_TO_WORKERS_ENABLED"
8589
SYSTEM_METRICS_INSTRUMENTATION_SCOPE_NAME = "opentelemetry.instrumentation.system_metrics"
8690
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT = "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"
87-
XRAY_OTLP_ENDPOINT_PATTERN = r"https://xray\.([a-z0-9-]+)\.amazonaws\.com/v1/traces$"
91+
OTEL_EXPORTER_OTLP_LOGS_ENDPOINT = "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT"
92+
OTEL_EXPORTER_OTLP_LOGS_HEADERS = "OTEL_EXPORTER_OTLP_LOGS_HEADERS"
93+
94+
AWS_TRACES_OTLP_ENDPOINT_PATTERN = r"https://xray\.([a-z0-9-]+)\.amazonaws\.com/v1/traces$"
95+
AWS_LOGS_OTLP_ENDPOINT_PATTERN = r"https://logs\.([a-z0-9-]+)\.amazonaws\.com/v1/logs$"
96+
97+
AWS_OTLP_LOGS_GROUP_HEADER = "x-aws-log-group"
98+
AWS_OTLP_LOGS_STREAM_HEADER = "x-aws-log-stream"
99+
88100
# UDP package size is not larger than 64KB
89101
LAMBDA_SPAN_EXPORT_BATCH_SIZE = 10
90102

@@ -160,6 +172,29 @@ def _initialize_components():
160172
_init_logging(log_exporters, resource)
161173

162174

175+
def _init_logging(
176+
exporters: Dict[str, Type[LogExporter]],
177+
resource: Resource = None,
178+
):
179+
180+
# Provides a default OTLP log exporter when none is specified.
181+
# This is the behavior for the logs exporters for other languages.
182+
if not exporters:
183+
exporters = {"otlp": OTLPLogExporter}
184+
185+
provider = LoggerProvider(resource=resource)
186+
set_logger_provider(provider)
187+
188+
for _, exporter_class in exporters.items():
189+
exporter_args: Dict[str, any] = {}
190+
log_exporter = _customize_logs_exporter(exporter_class(**exporter_args), resource)
191+
provider.add_log_record_processor(BatchLogRecordProcessor(exporter=log_exporter))
192+
193+
handler = LoggingHandler(level=NOTSET, logger_provider=provider)
194+
195+
getLogger().addHandler(handler)
196+
197+
163198
def _init_tracing(
164199
exporters: Dict[str, Type[SpanExporter]],
165200
id_generator: IdGenerator = None,
@@ -177,7 +212,7 @@ def _init_tracing(
177212
for _, exporter_class in exporters.items():
178213
exporter_args: Dict[str, any] = {}
179214
span_exporter: SpanExporter = exporter_class(**exporter_args)
180-
span_exporter = _customize_exporter(span_exporter, resource)
215+
span_exporter = _customize_span_exporter(span_exporter, resource)
181216
trace_provider.add_span_processor(
182217
BatchSpanProcessor(span_exporter=span_exporter, max_export_batch_size=_span_export_batch_size())
183218
)
@@ -312,19 +347,22 @@ def _customize_sampler(sampler: Sampler) -> Sampler:
312347
return AlwaysRecordSampler(sampler)
313348

314349

315-
def _customize_exporter(span_exporter: SpanExporter, resource: Resource) -> SpanExporter:
350+
def _customize_span_exporter(span_exporter: SpanExporter, resource: Resource) -> SpanExporter:
351+
traces_endpoint = os.environ.get(OTEL_EXPORTER_OTLP_TRACES_ENDPOINT)
316352
if _is_lambda_environment():
317353
# Override OTLP http default endpoint to UDP
318-
if isinstance(span_exporter, OTLPSpanExporter) and os.getenv(OTEL_EXPORTER_OTLP_TRACES_ENDPOINT) is None:
354+
if isinstance(span_exporter, OTLPSpanExporter) and traces_endpoint is None:
319355
traces_endpoint = os.environ.get(AWS_XRAY_DAEMON_ADDRESS_CONFIG, "127.0.0.1:2000")
320356
span_exporter = OTLPUdpSpanExporter(endpoint=traces_endpoint)
321357

322-
if is_xray_otlp_endpoint(os.environ.get(OTEL_EXPORTER_OTLP_TRACES_ENDPOINT)):
323-
# TODO: Change this url once doc writer has added a section for using SigV4 without collector
324-
_logger.info("Detected using AWS OTLP XRay Endpoint.")
358+
if _is_aws_otlp_endpoint(traces_endpoint, "xray"):
359+
_logger.info("Detected using AWS OTLP Traces Endpoint.")
325360

326361
if isinstance(span_exporter, OTLPSpanExporter):
327-
span_exporter = OTLPAwsSpanExporter(endpoint=os.getenv(OTEL_EXPORTER_OTLP_TRACES_ENDPOINT))
362+
span_exporter = OTLPSpanExporter(
363+
endpoint=traces_endpoint,
364+
session=AwsAuthSession(traces_endpoint.split(".")[1], "xray"),
365+
)
328366

329367
else:
330368
_logger.warning(
@@ -338,6 +376,30 @@ def _customize_exporter(span_exporter: SpanExporter, resource: Resource) -> Span
338376
return AwsMetricAttributesSpanExporterBuilder(span_exporter, resource).build()
339377

340378

379+
def _customize_logs_exporter(log_exporter: LogExporter, resource: Resource) -> LogExporter:
380+
logs_endpoint = os.environ.get(OTEL_EXPORTER_OTLP_LOGS_ENDPOINT)
381+
382+
if _is_aws_otlp_endpoint(logs_endpoint, "logs"):
383+
_logger.info("Detected using AWS OTLP Logs Endpoint.")
384+
385+
if isinstance(log_exporter, OTLPLogExporter) and _validate_logs_headers():
386+
# Setting default compression mode to Gzip as this is the behavior in upstream's
387+
# collector otlp http exporter:
388+
# https://github.com/open-telemetry/opentelemetry-collector/tree/main/exporter/otlphttpexporter
389+
return OTLPLogExporter(
390+
endpoint=logs_endpoint,
391+
compression=Compression.Gzip,
392+
session=AwsAuthSession(logs_endpoint.split(".")[1], "logs"),
393+
)
394+
395+
_logger.warning(
396+
"Improper configuration see: please export/set "
397+
"OTEL_EXPORTER_OTLP_LOGS_PROTOCOL=http/protobuf and OTEL_LOGS_EXPORTER=otlp"
398+
)
399+
400+
return log_exporter
401+
402+
341403
def _customize_span_processors(provider: TracerProvider, resource: Resource) -> None:
342404
# Add LambdaSpanProcessor to list of processors regardless of application signals.
343405
if _is_lambda_environment():
@@ -458,12 +520,48 @@ def _is_lambda_environment():
458520
return AWS_LAMBDA_FUNCTION_NAME_CONFIG in os.environ
459521

460522

461-
def is_xray_otlp_endpoint(otlp_endpoint: str = None) -> bool:
462-
"""Is the given endpoint the XRay OTLP endpoint?"""
523+
def _is_aws_otlp_endpoint(otlp_endpoint: str = None, service: str = "xray") -> bool:
524+
"""Is the given endpoint an AWS OTLP endpoint?"""
525+
526+
pattern = AWS_TRACES_OTLP_ENDPOINT_PATTERN if service == "xray" else AWS_LOGS_OTLP_ENDPOINT_PATTERN
527+
463528
if not otlp_endpoint:
464529
return False
465530

466-
return bool(re.match(XRAY_OTLP_ENDPOINT_PATTERN, otlp_endpoint.lower()))
531+
return bool(re.match(pattern, otlp_endpoint.lower()))
532+
533+
534+
def _validate_logs_headers() -> bool:
535+
"""Checks if x-aws-log-group and x-aws-log-stream are present in the headers in order to send logs to
536+
AWS OTLP Logs endpoint."""
537+
538+
logs_headers = os.environ.get(OTEL_EXPORTER_OTLP_LOGS_HEADERS)
539+
540+
if not logs_headers:
541+
_logger.warning(
542+
"Improper configuration: Please configure the environment variable OTEL_EXPORTER_OTLP_LOGS_HEADERS "
543+
"to include x-aws-log-group and x-aws-log-stream"
544+
)
545+
return False
546+
547+
filtered_log_headers_count = 0
548+
549+
for pair in logs_headers.split(","):
550+
if "=" in pair:
551+
split = pair.split("=", 1)
552+
key = split[0]
553+
value = split[1]
554+
if key in (AWS_OTLP_LOGS_GROUP_HEADER, AWS_OTLP_LOGS_STREAM_HEADER) and value:
555+
filtered_log_headers_count += 1
556+
557+
if filtered_log_headers_count != 2:
558+
_logger.warning(
559+
"Improper configuration: Please configure the environment variable OTEL_EXPORTER_OTLP_LOGS_HEADERS "
560+
"to have values for x-aws-log-group and x-aws-log-stream"
561+
)
562+
return False
563+
564+
return True
467565

468566

469567
def _get_metric_export_interval():
Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
import logging
5+
6+
import requests
7+
8+
from amazon.opentelemetry.distro._utils import is_installed
9+
10+
_logger = logging.getLogger(__name__)
11+
12+
13+
class AwsAuthSession(requests.Session):
14+
"""
15+
A custom requests Session that adds AWS SigV4 authentication to HTTP requests.
16+
17+
This class extends the standard requests.Session to automatically sign requests
18+
with AWS Signature Version 4 (SigV4) authentication. It's specifically designed
19+
for use with the OpenTelemetry Logs and Traces exporters that send data to AWS OTLP endpoints:
20+
X-Ray (traces) and CloudWatch Logs.
21+
22+
The session requires botocore to be installed for signing headers. If botocore
23+
is not available, the session will fall back to standard unauthenticated requests
24+
and log an error message.
25+
26+
Usage:
27+
session = AwsAuthSession(aws_region="us-west-2", service="logs")
28+
response = session.request("POST", "https://logs.us-west-2.amazonaws.com/v1/logs",
29+
data=payload, headers=headers)
30+
31+
Args:
32+
aws_region (str): The AWS region to use for signing (e.g., "us-east-1")
33+
service (str): The AWS service name for signing (e.g., "logs" or "xray")
34+
"""
35+
36+
def __init__(self, aws_region, service):
37+
38+
self._has_required_dependencies = False
39+
40+
# Requires botocore to be installed to sign the headers. However,
41+
# some users might not need to use this authenticator. In order not conflict
42+
# with existing behavior, we check for botocore before initializing this exporter.
43+
44+
if aws_region and service and is_installed("botocore"):
45+
# pylint: disable=import-outside-toplevel
46+
from botocore import auth, awsrequest, session
47+
48+
self._boto_auth = auth
49+
self._boto_aws_request = awsrequest
50+
self._boto_session = session.Session()
51+
52+
self._aws_region = aws_region
53+
self._service = service
54+
self._has_required_dependencies = True
55+
56+
else:
57+
_logger.error(
58+
"botocore is required to enable SigV4 Authentication. Please install it using `pip install botocore`",
59+
)
60+
61+
super().__init__()
62+
63+
def request(self, method, url, *args, data=None, headers=None, **kwargs):
64+
if self._has_required_dependencies:
65+
66+
credentials = self._boto_session.get_credentials()
67+
68+
if credentials is not None:
69+
signer = self._boto_auth.SigV4Auth(credentials, self._service, self._aws_region)
70+
71+
request = self._boto_aws_request.AWSRequest(
72+
method="POST",
73+
url=url,
74+
data=data,
75+
headers={"Content-Type": "application/x-protobuf"},
76+
)
77+
78+
try:
79+
signer.add_auth(request)
80+
81+
if headers is None:
82+
headers = {}
83+
84+
headers.update(dict(request.headers))
85+
86+
except Exception as signing_error: # pylint: disable=broad-except
87+
_logger.error("Failed to sign request: %s", signing_error)
88+
89+
return super().request(method=method, url=url, *args, data=data, headers=headers, **kwargs)

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/otlp_aws_span_exporter.py

Lines changed: 0 additions & 100 deletions
This file was deleted.

0 commit comments

Comments
 (0)