Skip to content

Commit 99ed284

Browse files
authored
Support export OTel metrics to be EMF in Standard Output in Lambda (#231)
*Issue #, if available:* - Currently, ADOT cannot exports metrics in AWS Lambda environment because Lambda freezes the execution environment asap after invocation. - The existing EMF Exporter only outputs to CW Logs *Description of changes:* - Sync AWS Lambda patch to forceFlush metrics after Lambda function invocation. - Introduce ConsoleEMFExporter, which exports OTel metrics to standard output using the CloudWatch Embedded Metric Format (EMF). In the Lambda environment, logs written to standard output are automatically forwarded to CloudWatch by Lambda's built-in logging agent, so this is an efficient way to export OTel metrics in Lambda to CW as EMF. *Testing:* ``` START RequestId: fc230af0-020a-429b-8a3d-b7c0a011a777 Version: $LATEST {"level":50,"time":1753469912921,"pid":2,"hostname":"169.254.67.97","msg":"/histogram endpoint 0"} 2025-07-25T18:58:33.141Z fc230af0-020a-429b-8a3d-b7c0a011a777 INFO {"_aws":{"Timestamp":1753469913043,"CloudWatchMetrics":[{"Namespace":"test_adot_namespace2","Metrics":[{"Name":"histogram.counter","Unit":"Milliseconds"}],"Dimensions":[["histogramKey1","histogramKey2"]]}]},"Version":"1","otel.resource.service.name":"adot-lambda-logs-emf-support-testing","otel.resource.telemetry.sdk.language":"nodejs","otel.resource.telemetry.sdk.name":"opentelemetry","otel.resource.telemetry.sdk.version":"1.30.1","otel.resource.telemetry.auto.version":"0.6.0-dev0-aws","otel.resource.cloud.region":"us-west-1","otel.resource.cloud.provider":"aws","otel.resource.faas.name":"adot-lambda-logs-emf-support-testing","otel.resource.faas.version":"$LATEST","otel.resource.faas.instance":"2025/07/25/[$LATEST]990958db5577472a8b72f185f040b136","otel.resource.aws.log.group.names":"/aws/lambda/adot-lambda-logs-emf-support-testing","histogram.counter":{"Values":[2.997969481393705,3.9945921121710146,0],"Counts":[1,2,2],"Count":5,"Sum":11,"Max":4,"Min":0},"histogramKey1":"histogramValue1","histogramKey2":"histogramValue2"} END RequestId: fc230af0-020a-429b-8a3d-b7c0a011a777 REPORT RequestId: fc230af0-020a-429b-8a3d-b7c0a011a777 Duration: 399.18 ms Billed Duration: 400 ms Memory Size: 128 MB Max Memory Used: 127 MB Init Duration: 1426.89 ms XRAY TraceId: 1-6883d3d7-48a03512232241fa27fe6529 SegmentId: ed5a5f628350482b Sampled: true ``` By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.
1 parent 629faf5 commit 99ed284

File tree

10 files changed

+878
-627
lines changed

10 files changed

+878
-627
lines changed

aws-distro-opentelemetry-node-autoinstrumentation/src/aws-opentelemetry-configurator.ts

Lines changed: 44 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,8 @@ import { BaggageSpanProcessor } from '@opentelemetry/baggage-span-processor';
8080
import { logs } from '@opentelemetry/api-logs';
8181
import { AWS_ATTRIBUTE_KEYS } from './aws-attribute-keys';
8282
import { AwsCloudWatchOtlpBatchLogRecordProcessor } from './exporter/otlp/aws/logs/aws-cw-otlp-batch-log-record-processor';
83+
import { ConsoleEMFExporter } from './exporter/aws/metrics/console-emf-exporter';
84+
import { EMFExporterBase } from './exporter/aws/metrics/emf-exporter-base';
8385

8486
const AWS_TRACES_OTLP_ENDPOINT_PATTERN = '^https://xray\\.([a-z0-9-]+)\\.amazonaws\\.com/v1/traces$';
8587
const AWS_LOGS_OTLP_ENDPOINT_PATTERN = '^https://logs\\.([a-z0-9-]+)\\.amazonaws\\.com/v1/logs$';
@@ -382,14 +384,17 @@ export class AwsOpentelemetryConfigurator {
382384
}
383385

384386
private customizeMetricReader(isEmfEnabled: boolean) {
387+
let exporter: PushMetricExporter | undefined = undefined;
388+
385389
if (isEmfEnabled) {
386-
const emfExporter = createEmfExporter();
387-
if (emfExporter) {
388-
const periodicExportingMetricReader = new PeriodicExportingMetricReader({
389-
exporter: emfExporter,
390-
});
391-
this.metricReader = periodicExportingMetricReader;
392-
}
390+
exporter = createEmfExporter();
391+
}
392+
393+
if (exporter) {
394+
const periodicExportingMetricReader = new PeriodicExportingMetricReader({
395+
exporter: exporter,
396+
});
397+
this.metricReader = periodicExportingMetricReader;
393398
}
394399
}
395400

@@ -523,8 +528,7 @@ export class AwsLoggerProcessorProvider {
523528
return exporters.map(exporter => {
524529
if (exporter instanceof ConsoleLogRecordExporter) {
525530
return new SimpleLogRecordProcessor(exporter);
526-
}
527-
if (exporter instanceof OTLPAwsLogExporter && isAgentObservabilityEnabled()) {
531+
} else if (exporter instanceof OTLPAwsLogExporter && isAgentObservabilityEnabled()) {
528532
return new AwsCloudWatchOtlpBatchLogRecordProcessor(exporter);
529533
}
530534
return new BatchLogRecordProcessor(exporter);
@@ -961,15 +965,17 @@ export function validateAndFetchLogsHeader(): OtlpLogHeaderSetting {
961965
const logHeaders = process.env.OTEL_EXPORTER_OTLP_LOGS_HEADERS;
962966

963967
if (!logHeaders) {
964-
diag.warn(
965-
'Missing required configuration: The environment variable OTEL_EXPORTER_OTLP_LOGS_HEADERS must be set with ' +
966-
`required headers ${AWS_OTLP_LOGS_GROUP_HEADER} and ${AWS_OTLP_LOGS_STREAM_HEADER}. ` +
967-
`Example: OTEL_EXPORTER_OTLP_LOGS_HEADERS="${AWS_OTLP_LOGS_GROUP_HEADER}=my-log-group,${AWS_OTLP_LOGS_STREAM_HEADER}=my-log-stream"`
968-
);
968+
if (!isLambdaEnvironment()) {
969+
diag.warn(
970+
'Missing required configuration: The environment variable OTEL_EXPORTER_OTLP_LOGS_HEADERS must be set with ' +
971+
`required headers ${AWS_OTLP_LOGS_GROUP_HEADER} and ${AWS_OTLP_LOGS_STREAM_HEADER}. ` +
972+
`Example: OTEL_EXPORTER_OTLP_LOGS_HEADERS="${AWS_OTLP_LOGS_GROUP_HEADER}=my-log-group,${AWS_OTLP_LOGS_STREAM_HEADER}=my-log-stream"`
973+
);
974+
}
969975
return {
970-
logGroup: '',
971-
logStream: '',
972-
namespace: '',
976+
logGroup: undefined,
977+
logStream: undefined,
978+
namespace: undefined,
973979
isValid: false,
974980
};
975981
}
@@ -1030,16 +1036,27 @@ export function checkEmfExporterEnabled(): boolean {
10301036
return true;
10311037
}
10321038

1033-
export function createEmfExporter(): AWSCloudWatchEMFExporter | undefined {
1034-
const headersResult = validateAndFetchLogsHeader();
1035-
if (!headersResult.isValid) {
1036-
return undefined;
1039+
/**
1040+
* Create the appropriate EMF exporter based on the environment and configuration.
1041+
*
1042+
* @returns {EMFExporterBase | undefined}
1043+
*/
1044+
export function createEmfExporter(): EMFExporterBase | undefined {
1045+
let exporter: EMFExporterBase | undefined = undefined;
1046+
const otlpLogHeaderSetting = validateAndFetchLogsHeader();
1047+
1048+
if (isLambdaEnvironment() && !otlpLogHeaderSetting.isValid) {
1049+
// Lambda without valid logs http headers - use Console EMF exporter
1050+
exporter = new ConsoleEMFExporter(otlpLogHeaderSetting.namespace);
1051+
} else if (otlpLogHeaderSetting.isValid) {
1052+
// Non-Lambda environment - use CloudWatch EMF exporter
1053+
// If headersResult.isValid is true, then headersResult.logGroup and headersResult.logStream are guaranteed to be strings
1054+
exporter = new AWSCloudWatchEMFExporter(
1055+
otlpLogHeaderSetting.namespace,
1056+
otlpLogHeaderSetting.logGroup as string,
1057+
otlpLogHeaderSetting.logStream as string
1058+
);
10371059
}
10381060

1039-
// If headersResult.isValid is true, then headersResult.logGroup and headersResult.logStream are guaranteed to be strings
1040-
return new AWSCloudWatchEMFExporter(
1041-
headersResult.namespace,
1042-
headersResult.logGroup as string,
1043-
headersResult.logStream as string
1044-
);
1061+
return exporter;
10451062
}

0 commit comments

Comments
 (0)