Skip to content

Commit 7151d66

Browse files
authored
Support forceFlush OTel Logs in Lambda Instrumentation. (#233)
*Issue #, if available:* *Description of changes:* Force flush logs in aws lambda instrumentation. Changes are based on design to force flush spans: https://github.com/open-telemetry/opentelemetry-js-contrib/blob/instrumentation-aws-lambda-v0.54.0/packages/instrumentation-aws-lambda/src/instrumentation.ts#L309-L324 *Testing:* Env Vars: ```sh AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-instrument OTEL_EXPORTER_OTLP_LOGS_HEADERS=x-aws-metric-namespace=test_adot_namespace3,x-aws-log-stream=my-stream-not-work OTEL_LOGS_EXPORTER=console OTEL_NODE_ENABLED_INSTRUMENTATIONS=aws-sdk,aws-lambda,http,pino ``` Lambda Handler: ```js import { metrics } from "@opentelemetry/api"; import { logs } from "@opentelemetry/api-logs"; import pino from "pino"; const logger = pino( pino.destination({ sync: true // Synchronous logging }) ); const meter = metrics.getMeter('dice-lib'); const histogram = meter.createHistogram('histogram.counter', {description: 'test_histogram_description', unit: 'ms', valueType: 0 }); export const handler = async (event) => { for (let i = 0; i < 5; i++) { let val = await getRandomNumber(0,5); // console.log(`/histogram endpoint ${val}`); logger.error(`/histogram endpoint ${val}`) histogram.record(val, { histogramKey1: 'histogramValue1', histogramKey2: 'histogramValue2' }); } const response = { statusCode: 200, body: JSON.stringify('Hello from Lambda!'), }; return response; }; async function getRandomNumber(min, max) { return Math.floor(Math.random() * (max - min) + min); } ``` ``` --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | timestamp | message | |---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | 1753725988746 | INIT_START Runtime Version: nodejs:22.v48 Runtime Version ARN: arn:aws:lambda:us-west-1::runtime:3319d7328c2e45f97764ca1c004269cab40102f3f1c188ec9a509d8e9c5db574 | | 1753725989027 | (node:2) ExperimentalWarning: `--experimental-loader` may be removed in the future; instead use `register()`: | | 1753725989027 | --import 'data:text/javascript,import { register } from "node:module"; import { pathToFileURL } from "node:url"; register("import-in-the-middle/hook.mjs", pathToFileURL("./"));' | | 1753725989027 | (Use `node --trace-warnings ...` to show where the warning was created) | | 1753725989803 | OTEL_TRACES_EXPORTER is empty. Using default otlp exporter. | | 1753725989806 | AWS Application Signals enabled. | | 1753725989808 | AWS Application Signals metrics export interval capped to 60000 | | 1753725989809 | Enabled batch unsampled span processor for Lambda environment. | | 1753725989812 | OTEL_METRICS_EXPORTER contains "none". Metric provider will not be initialized. | | 1753725989813 | Setting TraceProvider for instrumentations at the end of initialization | | 1753725989814 | AWS Distro of OpenTelemetry automatic instrumentation started successfully | | 1753725989815 | (node:2) [DEP0040] DeprecationWarning: The `punycode` module is deprecated. Please use a userland alternative instead. | | 1753725990219 | 2025-07-28T18:06:30.219Z undefined WARN Failed extracting version /var/task | | 1753725990227 | 2025-07-28T18:06:30.227Z undefined WARN AWS Lambda plans to remove support for callback-based function handlers starting with Node.js 24. You will need to update this function to use an async handler to use Node.js 24 or later. For more information and to provide feedback on this change, see https://github.com/aws/aws-lambda-nodejs-runtime-interface-client/issues/137. To disable this warning, set the AWS_LAMBDA_NODEJS_DISABLE_CALLBACK_WARNING environment variable. | | 1753725990239 | START RequestId: c421f663-0e83-4ec1-8c09-56f40ed74ea1 Version: $LATEST | | 1753725990342 | { | | 1753725990342 | resource: { | | 1753725990342 | attributes: { | | 1753725990342 | 'service.name': 'adot-lambda-logs-emf-support-testing', | | 1753725990342 | 'telemetry.sdk.language': 'nodejs', | | 1753725990342 | 'telemetry.sdk.name': 'opentelemetry', | | 1753725990342 | 'telemetry.sdk.version': '1.30.1', | | 1753725990342 | 'telemetry.auto.version': '0.6.0-dev0-aws', | | 1753725990342 | 'cloud.region': 'us-west-1', | | 1753725990342 | 'cloud.provider': 'aws', | | 1753725990342 | 'faas.name': 'adot-lambda-logs-emf-support-testing', | | 1753725990342 | 'faas.version': '$LATEST', | | 1753725990342 | 'faas.instance': '2025/07/28/[$LATEST]c8c585140d9b4cf4aef5c50de3c0328a', | | 1753725990342 | 'aws.log.group.names': '/aws/lambda/adot-lambda-logs-emf-support-testing' | | 1753725990342 | } | | 1753725990342 | }, | | 1753725990342 | instrumentationScope: { | | 1753725990342 | name: '@opentelemetry/instrumentation-pino', | | 1753725990342 | version: '0.46.1', | | 1753725990342 | schemaUrl: undefined | | 1753725990342 | }, | | 1753725990342 | timestamp: 1753725990304000, | | 1753725990342 | traceId: '6887bc2469a702443d9e988c3411ef70', | | 1753725990342 | spanId: 'bf082bff655a2682', | | 1753725990342 | traceFlags: 1, | | 1753725990342 | severityText: 'error', | | 1753725990342 | severityNumber: 17, | | 1753725990342 | body: '/histogram endpoint 0', | | 1753725990342 | attributes: {} | | 1753725990342 | } | | 1753725990343 | {"level":50,"time":1753725990304,"pid":2,"hostname":"169.254.14.25","trace_id":"6887bc2469a702443d9e988c3411ef70","span_id":"bf082bff655a2682","trace_flags":"01","msg":"/histogram endpoint 0"} | | 1753725990344 | { | | 1753725990344 | resource: { | | 1753725990344 | attributes: { | | 1753725990344 | 'service.name': 'adot-lambda-logs-emf-support-testing', | | 1753725990344 | 'telemetry.sdk.language': 'nodejs', | | 1753725990344 | 'telemetry.sdk.name': 'opentelemetry', | | 1753725990344 | 'telemetry.sdk.version': '1.30.1', | | 1753725990344 | 'telemetry.auto.version': '0.6.0-dev0-aws', | | 1753725990344 | 'cloud.region': 'us-west-1', | | 1753725990344 | 'cloud.provider': 'aws', | | 1753725990344 | 'faas.name': 'adot-lambda-logs-emf-support-testing', | | 1753725990344 | 'faas.version': '$LATEST', | | 1753725990344 | 'faas.instance': '2025/07/28/[$LATEST]c8c585140d9b4cf4aef5c50de3c0328a', | | 1753725990344 | 'aws.log.group.names': '/aws/lambda/adot-lambda-logs-emf-support-testing' | | 1753725990344 | } | | 1753725990344 | }, | | 1753725990344 | instrumentationScope: { | | 1753725990344 | name: '@opentelemetry/instrumentation-pino', | | 1753725990344 | version: '0.46.1', | | 1753725990344 | schemaUrl: undefined | | 1753725990344 | }, | | 1753725990344 | timestamp: 1753725990343000, | | 1753725990344 | traceId: '6887bc2469a702443d9e988c3411ef70', | | 1753725990344 | spanId: 'bf082bff655a2682', | | 1753725990344 | traceFlags: 1, | | 1753725990344 | severityText: 'error', | | 1753725990344 | severityNumber: 17, | | 1753725990344 | body: '/histogram endpoint 4', | | 1753725990344 | attributes: {} | | 1753725990344 | } | | 1753725990344 | {"level":50,"time":1753725990343,"pid":2,"hostname":"169.254.14.25","trace_id":"6887bc2469a702443d9e988c3411ef70","span_id":"bf082bff655a2682","trace_flags":"01","msg":"/histogram endpoint 4"} | | 1753725990344 | { | | 1753725990344 | resource: { | | 1753725990344 | attributes: { | | 1753725990344 | 'service.name': 'adot-lambda-logs-emf-support-testing', | | 1753725990344 | 'telemetry.sdk.language': 'nodejs', | | 1753725990344 | 'telemetry.sdk.name': 'opentelemetry', | | 1753725990344 | 'telemetry.sdk.version': '1.30.1', | | 1753725990344 | 'telemetry.auto.version': '0.6.0-dev0-aws', | | 1753725990344 | 'cloud.region': 'us-west-1', | | 1753725990344 | 'cloud.provider': 'aws', | | 1753725990344 | 'faas.name': 'adot-lambda-logs-emf-support-testing', | | 1753725990344 | 'faas.version': '$LATEST', | | 1753725990344 | 'faas.instance': '2025/07/28/[$LATEST]c8c585140d9b4cf4aef5c50de3c0328a', | | 1753725990344 | 'aws.log.group.names': '/aws/lambda/adot-lambda-logs-emf-support-testing' | | 1753725990344 | } | | 1753725990344 | }, | | 1753725990344 | instrumentationScope: { | | 1753725990344 | name: '@opentelemetry/instrumentation-pino', | | 1753725990344 | version: '0.46.1', | | 1753725990344 | schemaUrl: undefined | | 1753725990344 | }, | | 1753725990344 | timestamp: 1753725990344000, | | 1753725990344 | traceId: '6887bc2469a702443d9e988c3411ef70', | | 1753725990344 | spanId: 'bf082bff655a2682', | | 1753725990344 | traceFlags: 1, | | 1753725990344 | severityText: 'error', | | 1753725990344 | severityNumber: 17, | | 1753725990344 | body: '/histogram endpoint 3', | | 1753725990344 | attributes: {} | | 1753725990344 | } | | 1753725990344 | {"level":50,"time":1753725990344,"pid":2,"hostname":"169.254.14.25","trace_id":"6887bc2469a702443d9e988c3411ef70","span_id":"bf082bff655a2682","trace_flags":"01","msg":"/histogram endpoint 3"} | | 1753725990345 | { | | 1753725990345 | resource: { | | 1753725990345 | attributes: { | | 1753725990345 | 'service.name': 'adot-lambda-logs-emf-support-testing', | | 1753725990345 | 'telemetry.sdk.language': 'nodejs', | | 1753725990345 | 'telemetry.sdk.name': 'opentelemetry', | | 1753725990345 | 'telemetry.sdk.version': '1.30.1', | | 1753725990345 | 'telemetry.auto.version': '0.6.0-dev0-aws', | | 1753725990345 | 'cloud.region': 'us-west-1', | | 1753725990345 | 'cloud.provider': 'aws', | | 1753725990345 | 'faas.name': 'adot-lambda-logs-emf-support-testing', | | 1753725990345 | 'faas.version': '$LATEST', | | 1753725990345 | 'faas.instance': '2025/07/28/[$LATEST]c8c585140d9b4cf4aef5c50de3c0328a', | | 1753725990345 | 'aws.log.group.names': '/aws/lambda/adot-lambda-logs-emf-support-testing' | | 1753725990345 | } | | 1753725990345 | }, | | 1753725990345 | instrumentationScope: { | | 1753725990345 | name: '@opentelemetry/instrumentation-pino', | | 1753725990345 | version: '0.46.1', | | 1753725990345 | schemaUrl: undefined …
1 parent 99ed284 commit 7151d66

File tree

4 files changed

+178
-3
lines changed

4 files changed

+178
-3
lines changed

aws-distro-opentelemetry-node-autoinstrumentation/src/patches/instrumentation-patch.ts

Lines changed: 49 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ import { SecretsManagerServiceExtension } from './aws/services/secretsmanager';
3636
import { StepFunctionsServiceExtension } from './aws/services/step-functions';
3737
import type { AwsLambdaInstrumentation } from '@opentelemetry/instrumentation-aws-lambda';
3838
import type { Command as AwsV3Command } from '@aws-sdk/types';
39+
import { LoggerProvider } from '@opentelemetry/api-logs';
3940

4041
export const traceContextEnvironmentKey = '_X_AMZN_TRACE_ID';
4142
export const AWSXRAY_TRACE_ID_HEADER_CAPITALIZED = 'X-Amzn-Trace-Id';
@@ -251,11 +252,49 @@ function patchLambdaServiceExtension(lambdaServiceExtension: any): void {
251252
}
252253
}
253254

254-
// Override the upstream private _endSpan method to remove the unnecessary metric force-flush error message
255-
// https://github.com/open-telemetry/opentelemetry-js-contrib/blob/main/plugins/node/opentelemetry-instrumentation-aws-lambda/src/instrumentation.ts#L358-L398
255+
export type ExtendedAwsLambdaInstrumentation = AwsLambdaInstrumentation & {
256+
_setLoggerProvider: (loggerProvider: LoggerProvider) => void;
257+
_logForceFlusher?: () => Promise<void>;
258+
_logForceFlush: (loggerProvider: LoggerProvider) => any;
259+
};
260+
261+
// Patch AWS Lambda Instrumentation
262+
// 1. Override the upstream private _endSpan method to remove the unnecessary metric force-flush error message
263+
// https://github.com/open-telemetry/opentelemetry-js-contrib/blob/main/plugins/node/opentelemetry-instrumentation-aws-lambda/src/instrumentation.ts#L358-L398
264+
// 2. Support setting logger provider and force flushing logs
256265
function patchAwsLambdaInstrumentation(instrumentation: Instrumentation): void {
257266
if (instrumentation) {
258-
(instrumentation as AwsLambdaInstrumentation)['_endSpan'] = function (
267+
const _setLoggerProvider = (instrumentation as ExtendedAwsLambdaInstrumentation)['setLoggerProvider'];
268+
(instrumentation as ExtendedAwsLambdaInstrumentation)['_setLoggerProvider'] = _setLoggerProvider;
269+
(instrumentation as ExtendedAwsLambdaInstrumentation)['_logForceFlusher'] = undefined;
270+
271+
instrumentation['setLoggerProvider'] = function (loggerProvider: LoggerProvider) {
272+
(this as ExtendedAwsLambdaInstrumentation)['_setLoggerProvider'](loggerProvider);
273+
(this as ExtendedAwsLambdaInstrumentation)['_logForceFlusher'] = (this as ExtendedAwsLambdaInstrumentation)[
274+
'_logForceFlush'
275+
](loggerProvider);
276+
};
277+
278+
(instrumentation as ExtendedAwsLambdaInstrumentation)['_logForceFlush'] = function (
279+
loggerProvider: LoggerProvider
280+
) {
281+
if (!loggerProvider) return undefined;
282+
283+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
284+
let currentProvider: any = loggerProvider;
285+
286+
if (typeof currentProvider.getDelegate === 'function') {
287+
currentProvider = currentProvider.getDelegate();
288+
}
289+
290+
if (typeof currentProvider.forceFlush === 'function') {
291+
return currentProvider.forceFlush.bind(currentProvider);
292+
}
293+
294+
return undefined;
295+
};
296+
297+
(instrumentation as ExtendedAwsLambdaInstrumentation)['_endSpan'] = function (
259298
span: Span,
260299
err: string | Error | null | undefined,
261300
callback: () => void
@@ -294,6 +333,13 @@ function patchAwsLambdaInstrumentation(instrumentation: Instrumentation): void {
294333
'Metrics may not be exported for the lambda function because we are not force flushing before callback.'
295334
);
296335
}
336+
if (this['_logForceFlusher']) {
337+
flushers.push(this['_logForceFlusher']());
338+
} else {
339+
diag.debug(
340+
'Logs may not be exported for the lambda function because we are not force flushing before callback.'
341+
);
342+
}
297343

298344
Promise.all(flushers).then(callback, callback);
299345
};

aws-distro-opentelemetry-node-autoinstrumentation/src/register.ts

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ if (process.env.OTEL_TRACES_SAMPLER === 'xray') {
1414
}
1515

1616
import { diag, DiagConsoleLogger, metrics, trace } from '@opentelemetry/api';
17+
import { logs } from '@opentelemetry/api-logs';
1718
import { getNodeAutoInstrumentations, InstrumentationConfigMap } from '@opentelemetry/auto-instrumentations-node';
1819
import { Instrumentation } from '@opentelemetry/instrumentation';
1920
import * as opentelemetry from '@opentelemetry/sdk-node';
@@ -137,6 +138,9 @@ try {
137138
for (const instrumentation of instrumentations) {
138139
instrumentation.setTracerProvider(trace.getTracerProvider());
139140
instrumentation.setMeterProvider(metrics.getMeterProvider());
141+
if (instrumentation.setLoggerProvider) {
142+
instrumentation.setLoggerProvider(logs.getLoggerProvider());
143+
}
140144
}
141145

142146
diag.debug(`Environment variable OTEL_PROPAGATORS is set to '${process.env.OTEL_PROPAGATORS}'`);

aws-distro-opentelemetry-node-autoinstrumentation/test/patches/instrumentation-patch.test.ts

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import {
2626
applyInstrumentationPatches,
2727
AWSXRAY_TRACE_ID_HEADER_CAPITALIZED,
2828
customExtractor,
29+
ExtendedAwsLambdaInstrumentation,
2930
headerGetter,
3031
} from './../../src/patches/instrumentation-patch';
3132
import * as sinon from 'sinon';
@@ -37,6 +38,7 @@ import * as nock from 'nock';
3738
import { ReadableSpan, Span as SDKSpan } from '@opentelemetry/sdk-trace-base';
3839
import { getTestSpans } from '@opentelemetry/contrib-test-utils';
3940
import { instrumentationConfigs } from '../../src/register';
41+
import { LoggerProvider } from '@opentelemetry/api-logs';
4042

4143
// It is assumed that bedrock.test.ts has already registered the
4244
// necessary instrumentations for testing by calling:
@@ -673,6 +675,124 @@ describe('InstrumentationPatchTest', () => {
673675
});
674676
});
675677
});
678+
679+
describe('AwsLambdaInstrumentationPatchTest', () => {
680+
let awsLambdaInstrumentation: ExtendedAwsLambdaInstrumentation;
681+
beforeEach(() => {
682+
const instrumentationsToTest = [
683+
new AwsLambdaInstrumentation(instrumentationConfigs['@opentelemetry/instrumentation-aws-lambda']),
684+
];
685+
applyInstrumentationPatches(instrumentationsToTest);
686+
awsLambdaInstrumentation = instrumentationsToTest[0] as ExtendedAwsLambdaInstrumentation;
687+
});
688+
689+
afterEach(() => {
690+
sinon.restore();
691+
});
692+
693+
it('Tests setLoggerProvider method', () => {
694+
const resolvedPromise = Promise.resolve();
695+
const mockLoggerProvider = {
696+
getDelegate: () => ({
697+
forceFlush: () => resolvedPromise,
698+
}),
699+
getLogger: () => {
700+
return {
701+
emit: () => {},
702+
};
703+
},
704+
};
705+
706+
awsLambdaInstrumentation.setLoggerProvider(mockLoggerProvider as LoggerProvider);
707+
expect(awsLambdaInstrumentation['_logForceFlusher']!()).toBe(resolvedPromise);
708+
});
709+
710+
it('Tests _logForceFlush with provider that has getDelegate', () => {
711+
const mockForceFlush = sinon.stub().resolves();
712+
const mockLoggerProvider = {
713+
getDelegate: () => ({
714+
forceFlush: mockForceFlush,
715+
}),
716+
getLogger: () => {
717+
return {
718+
emit: () => {},
719+
};
720+
},
721+
};
722+
723+
const flusher = awsLambdaInstrumentation['_logForceFlush'](mockLoggerProvider as LoggerProvider);
724+
expect(flusher).toBeDefined();
725+
flusher?.();
726+
expect(mockForceFlush.called).toBeTruthy();
727+
});
728+
729+
it('Tests _logForceFlush with provider that has direct forceFlush', () => {
730+
const mockForceFlush = sinon.stub().resolves();
731+
const mockLoggerProvider = {
732+
forceFlush: mockForceFlush,
733+
getLogger: () => {
734+
return {
735+
emit: () => {},
736+
};
737+
},
738+
};
739+
740+
const flusher = awsLambdaInstrumentation['_logForceFlush'](mockLoggerProvider as LoggerProvider);
741+
expect(flusher).toBeDefined();
742+
flusher?.();
743+
expect(mockForceFlush.called).toBeTruthy();
744+
});
745+
746+
it('Tests _logForceFlush with undefined provider', () => {
747+
const flusher = awsLambdaInstrumentation['_logForceFlush'](undefined as unknown as LoggerProvider);
748+
expect(flusher).toBeUndefined();
749+
});
750+
751+
it('Tests _endSpan with all flushers', done => {
752+
const mockSpan: Span = sinon.createStubInstance(SDKSpan);
753+
754+
// Setup mock flushers
755+
const mockTraceFlush = sinon.stub().resolves();
756+
const mockMetricFlush = sinon.stub().resolves();
757+
const mockLogFlush = sinon.stub().resolves();
758+
759+
awsLambdaInstrumentation['_traceForceFlusher'] = mockTraceFlush;
760+
awsLambdaInstrumentation['_metricForceFlusher'] = mockMetricFlush;
761+
awsLambdaInstrumentation['_logForceFlusher'] = mockLogFlush;
762+
763+
awsLambdaInstrumentation['_endSpan'](mockSpan, null, () => {
764+
expect(mockTraceFlush.called).toBeTruthy();
765+
expect(mockMetricFlush.called).toBeTruthy();
766+
expect(mockLogFlush.called).toBeTruthy();
767+
done();
768+
});
769+
});
770+
771+
it('Tests _endSpan handles missing flushers gracefully', done => {
772+
const mockSpan: Span = sinon.createStubInstance(SDKSpan);
773+
const mockDiag = sinon.spy(diag, 'debug');
774+
const mockDiagError = sinon.spy(diag, 'error');
775+
776+
awsLambdaInstrumentation['_endSpan'](mockSpan, null, () => {
777+
expect(
778+
mockDiagError.calledWith(
779+
'Spans may not be exported for the lambda function because we are not force flushing before callback.'
780+
)
781+
).toBeTruthy();
782+
expect(
783+
mockDiag.calledWith(
784+
'Metrics may not be exported for the lambda function because we are not force flushing before callback.'
785+
)
786+
).toBeTruthy();
787+
expect(
788+
mockDiag.calledWith(
789+
'Logs may not be exported for the lambda function because we are not force flushing before callback.'
790+
)
791+
).toBeTruthy();
792+
done();
793+
});
794+
});
795+
});
676796
});
677797

678798
describe('customExtractor', () => {

lambda-layer/packages/layer/scripts/otel-instrument

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,11 @@ if [ -z "${OTEL_METRICS_EXPORTER}" ]; then
8181
export OTEL_METRICS_EXPORTER="awsemf";
8282
fi
8383

84+
# - Disable logs exporter by default
85+
if [ -z "${OTEL_LOGS_EXPORTER}" ]; then
86+
export OTEL_LOGS_EXPORTER="none";
87+
fi
88+
8489
# - Append Lambda Resource Attributes to OTel Resource Attribute List
8590
if [ -z "${OTEL_RESOURCE_ATTRIBUTES}" ]; then
8691
export OTEL_RESOURCE_ATTRIBUTES=$LAMBDA_RESOURCE_ATTRIBUTES;

0 commit comments

Comments
 (0)