Skip to content
Open
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 42 additions & 0 deletions packages/opentelemetry-node/lib/central-config.js
Original file line number Diff line number Diff line change
Expand Up @@ -420,6 +420,46 @@ const REMOTE_CONFIG_HANDLERS = [
return null;
},
},

{
keys: ['sampling_rate'],
setter: (config, sdkInfo) => {
if (!sdkInfo.sampler) {
log.info(
`central-config: ignoring "sampling_rate" because non-default sampler in use`
);
return null;
}

const rawRate = config['sampling_rate'];
let valRate;
switch (typeof rawRate) {
case 'undefined':
valRate = initialConfig.sampling_rate;
break;
case 'number':
valRate = rawRate;
break;
case 'string':
valRate = Number(rawRate);
if (isNaN(valRate)) {
return `unknown 'sampling_rate' value: "${rawRate}"`;
}
break;
default:
return `unknown 'sampling_rate' value type: ${typeof rawRate} (${rawRate})`;
}

if (valRate < 0 || valRate > 1) {
return `'sampling_rate' value must be between 0 and 1: ${valRate}`;
}

sdkInfo.sampler.setRatio(valRate);
log.info(`central-config: set "sampling_rate" to "${valRate}"`);

return null;
},
},
];

/**
Expand Down Expand Up @@ -598,8 +638,10 @@ function setupCentralConfig(sdkInfo) {
CC_LOGGING_LEVEL_FROM_LUGGITE_LEVEL[
luggite.nameFromLevel[log.level()] ?? DEFAULT_LOG_LEVEL
];
initialConfig.sampling_rate = sdkInfo.samplingRate;
initialConfig.send_traces = !sdkInfo.contextPropagationOnly;
log.debug({initialConfig}, 'initial central config values');
lastAppliedConfig = {...initialConfig};
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I couldn't test the undefined behavior without this, and it seems to intuitively make sense to treat the initial config as the first applied config. What do you think?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This results some (the ones with initialConfig entries) of the central config setters being run when receiving a first config. For example, if I run an OpAMP server that will send an empty config:

% cd packages/mockopampserver
% cat config-empty.json
{}
% node lib/cli.js -F [email protected] | bunyan
...

Then run the examples/central-config.js against that, then (when central config is first received) we get unnecessary re-application of default values, and some slightly misleading logging:

% ELASTIC_OTEL_OPAMP_ENDPOINT=localhost:4320 \
        ELASTIC_OTEL_EXPERIMENTAL_OPAMP_HEARTBEAT_INTERVAL=5000 \
        node --import @elastic/opentelemetry-node central-config.js

{"name":"elastic-otel-node","level":30,"preamble":true,"distroVersion":"1.5.0","env":{"os":"darwin 24.6.0","arch":"arm64","runtime":"Node.js v22.17.1"},"config":{"logLevel":"info"},"msg":"start EDOT Node.js","time":"2025-10-17T20:10:08.455Z"}
{"name":"elastic-otel-node","level":40,"msg":"Unsupported OTLP metrics protocol: \"undefined\". Using http/protobuf.","time":"2025-10-17T20:10:08.476Z"}
{"name":"elastic-otel-node","level":30,"msg":"central-config: reset \"logging_level\" to \"info\"","time":"2025-10-17T20:10:08.529Z"}
{"name":"elastic-otel-node","level":30,"msg":"central-config: reset \"send_traces\" to \"true\"","time":"2025-10-17T20:10:08.530Z"}
{"name":"elastic-otel-node","level":30,"msg":"central-config: reset \"sampling_rate\" to \"1\"","time":"2025-10-17T20:10:08.530Z"}
{"name":"elastic-otel-node","level":30,"config":{},"appliedKeys":["logging_level","send_traces","sampling_rate"],"msg":"successfully applied remote config","time":"2025-10-17T20:10:08.530Z"}

...

I understand that this means the sampling_rate: undefined case cannot be tested with the "central-config-gen-telelemetry.js" fixture. FWIW, the logging_level: undefined case is being tested with a custom fixture that dynamically changes the central config during its run:

if (process.env.ELASTIC_OTEL_OPAMP_ENDPOINT) {
await setElasticConfig({logging_level: 'debug'});
await barrierOpAMPClientDiagEvents(3, [DIAG_CH_SEND_SUCCESS]);
}
// 3. Now the `diag.debug` should result in an emitted record.
diag.verbose('verbose2');
diag.debug('debug2');
diag.info('info2');
diag.warn('warn2');
diag.error('error2');
// 4. Set central config to empty config, to simulate an Agent Configuration
// in Kibana being removed. We expect the EDOT Node.js SDK to reset back
// to the default (info) log level.
if (process.env.ELASTIC_OTEL_OPAMP_ENDPOINT) {
await setElasticConfig({});
await barrierOpAMPClientDiagEvents(3, [DIAG_CH_SEND_SUCCESS]);
}

Personally I'm fine with not having an automated test for the undefined case here: complexity/code-size of the tests is a trade-off. Or, I wouldn't object to a test case doing live-updating of the central config (as in the logging_level example).

For consideration, I'll push a change that removes this and drops the test case for undefined.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For consideration, I'll push a change that removes this and drops the test case for undefined.

commit ee65525


const client = createOpAMPClient({
log,
Expand Down
89 changes: 89 additions & 0 deletions packages/opentelemetry-node/lib/sampler.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
/*
* Copyright Elasticsearch B.V. and contributors
* SPDX-License-Identifier: Apache-2.0
*/

const {
createCompositeSampler,
createComposableParentThresholdSampler,
createComposableTraceIDRatioBasedSampler,
} = require('@opentelemetry/sampler-composite');

/**
* @typedef {import('@opentelemetry/api').Attributes} Attributes
* @typedef {import('@opentelemetry/api').Context} Context
* @typedef {import('@opentelemetry/api').Link} Link
* @typedef {import('@opentelemetry/api').SpanKind} SpanKind
* @typedef {import('@opentelemetry/sdk-trace-base').Sampler} Sampler
* @typedef {import('@opentelemetry/sdk-trace-base').SamplingResult} SamplingResult
*/

/**
* EDOT default sampler, a parent-based ratio sampler which can have its ratio updated dynamically.
*
* @implements {Sampler}
*/
class DefaultSampler {
#delegate;

constructor(ratio = 1.0) {
this.#delegate = newSampler(ratio);
}

/**
* @param {Context} context
* @param {string} traceId
* @param {string} spanName
* @param {SpanKind} spanKind
* @param {Attributes} attributes
* @param {Link[]} links
* @returns {SamplingResult}
*/
shouldSample(context, traceId, spanName, spanKind, attributes, links) {
return this.#delegate.shouldSample(
context,
traceId,
spanName,
spanKind,
attributes,
links
);
}

/**
* @param {number} ratio
*/
setRatio(ratio) {
this.#delegate = newSampler(ratio);
}

toString() {
return this.#delegate.toString();
}
}

/**
* @param {number} ratio A number between 0 and 1 representing the sampling ratio.
*/
function newSampler(ratio) {
return createCompositeSampler(
createComposableParentThresholdSampler(
createComposableTraceIDRatioBasedSampler(ratio)
)
);
}

/**
* Creates a default EDOT sampler, which is a parent-based ratio sampler that can have
* its ratio updated dynamically by central config.
*
* @param {number} ratio
* @returns {Sampler} A ratio sampler which can have its ratio updated dynamically.
*/
function createDefaultSampler(ratio) {
return new DefaultSampler(ratio);
}

module.exports = {
createDefaultSampler,
};
25 changes: 25 additions & 0 deletions packages/opentelemetry-node/lib/sdk.js
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

const {
getBooleanFromEnv,
getNumberFromEnv,
getStringFromEnv,
getStringListFromEnv,
} = require('@opentelemetry/core');
Expand Down Expand Up @@ -45,10 +46,12 @@
setupDynConfExporters,
dynConfSpanExporters,
} = require('./dynconf');
const {createDefaultSampler} = require('./sampler');
const DISTRO_VERSION = require('../package.json').version;

/**
* @typedef {import('@opentelemetry/sdk-node').NodeSDKConfiguration} NodeSDKConfiguration
* @typedef {import('@opentelemetry/sdk-trace-base').Sampler} Sampler
*/

/**
Expand All @@ -66,7 +69,7 @@
try {
await shutdownFn();
} catch (err) {
console.warn('warning: error shutting down OTel SDK', err);

Check warning on line 72 in packages/opentelemetry-node/lib/sdk.js

View workflow job for this annotation

GitHub Actions / lint

Unexpected console statement
}
process.exit(128 + os.constants.signals.SIGTERM);
});
Expand All @@ -76,7 +79,7 @@
try {
await shutdownFn();
} catch (err) {
console.warn('warning: error shutting down OTel SDK', err);

Check warning on line 82 in packages/opentelemetry-node/lib/sdk.js

View workflow job for this annotation

GitHub Actions / lint

Unexpected console statement
}
});
}
Expand Down Expand Up @@ -241,6 +244,26 @@

const config = {...defaultConfig, ...cfg};

/** @type {Sampler} */
let sampler = undefined;
let samplingRate = 1.0;
if (!config.sampler && !getStringFromEnv('OTEL_TRACES_SAMPLER')) {
// If the user has not set a sampler via config or env var, use our default sampler.
// First get as string to differentiate between missing and invalid.
if (getStringFromEnv('OTEL_TRACES_SAMPLER_ARG')) {
const samplingRateArg = getNumberFromEnv('OTEL_TRACES_SAMPLER_ARG');
if (samplingRateArg === undefined) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should also validate the [0,1] range here, else an invalid value crashes:

% OTEL_TRACES_SAMPLER_ARG=-0.2 ELASTIC_OTEL_OPAMP_ENDPOINT=localhost:4320 \
        ELASTIC_OTEL_EXPERIMENTAL_OPAMP_HEARTBEAT_INTERVAL=5000 \
        node --import @elastic/opentelemetry-node central-config.js

/Users/trentm/el/elastic-otel-node2/packages/opentelemetry-node/node_modules/@opentelemetry/sampler-composite/build/src/traceidratio.js:26
            throw new Error(`Invalid sampling probability: ${ratio}. Must be between 0 and 1.`);
                  ^

Error: Invalid sampling probability: -0.2. Must be between 0 and 1.
    at new ComposableTraceIDRatioBasedSampler (/Users/trentm/el/elastic-otel-node2/packages/opentelemetry-node/node_modules/@opentelemetry/sampler-composite/build/src/traceidratio.js:26:19)
    at createComposableTraceIDRatioBasedSampler (/Users/trentm/el/elastic-otel-node2/packages/opentelemetry-node/node_modules/@opentelemetry/sampler-composite/build/src/traceidratio.js:58:12)
    at newSampler (/Users/trentm/el/elastic-otel-node2/packages/opentelemetry-node/lib/sampler.js:71:13)
    at new DefaultSampler (/Users/trentm/el/elastic-otel-node2/packages/opentelemetry-node/lib/sampler.js:30:26)
    at createDefaultSampler (/Users/trentm/el/elastic-otel-node2/packages/opentelemetry-node/lib/sampler.js:84:12)
    at startNodeSDK (/Users/trentm/el/elastic-otel-node2/packages/opentelemetry-node/lib/sdk.js:263:19)
    at file:///Users/trentm/el/elastic-otel-node2/packages/opentelemetry-node/import.mjs:24:5
    at ModuleJob.run (node:internal/modules/esm/module_job:329:25)
    at async onImport.tracePromise.__proto__ (node:internal/modules/esm/loader:644:26)
    at async asyncRunEntryPointWithESMLoader (node:internal/modules/run_main:112:9)

Node.js v22.17.1

log.warn(
`Invalid OTEL_TRACES_SAMPLER_ARG value: ${process.env.OTEL_TRACES_SAMPLER_ARG}. Using default sampling rate of ${samplingRate}`
);
} else {
samplingRate = samplingRateArg;
}
}
sampler = createDefaultSampler(samplingRate);
config.sampler = sampler;
}

// Some tricks to get a handle on noop signal providers, to be used for
// dynamic configuration.
const tracerProviderProxy = new api.ProxyTracerProvider();
Expand Down Expand Up @@ -306,6 +329,8 @@
noopTracerProvider,
// @ts-ignore: Ignore access of private _tracerProvider for now. (TODO)
sdkTracerProvider: sdk._tracerProvider,
sampler,
samplingRate,
contextPropagationOnly,
});

Expand Down
65 changes: 65 additions & 0 deletions packages/opentelemetry-node/package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions packages/opentelemetry-node/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@
"@opentelemetry/resource-detector-azure": "^0.14.0",
"@opentelemetry/resource-detector-container": "^0.7.0",
"@opentelemetry/resources": "^2.0.0",
"@opentelemetry/sampler-composite": "^0.206.0",
"@opentelemetry/sdk-logs": "^0.206.0",
"@opentelemetry/sdk-metrics": "^2.0.0",
"@opentelemetry/sdk-node": "^0.206.0",
Expand Down
53 changes: 53 additions & 0 deletions packages/opentelemetry-node/test/OTEL_TRACES_SAMPLER.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
/*
* Copyright Elasticsearch B.V. and contributors
* SPDX-License-Identifier: Apache-2.0
*/

const {test} = require('tape');
const {runTestFixtures} = require('./testutils');

/** @type {import('./testutils').TestFixture[]} */
const testFixtures = [
{
name: 'OTEL_TRACES_SAMPLER unset (default sampling of 100%)',
args: ['./fixtures/use-http-get.js'],
cwd: __dirname,
env: {
NODE_OPTIONS: '--import=@elastic/opentelemetry-node',
},
checkTelemetry: (t, col) => {
t.equal(col.sortedSpans.length, 1);
},
},
{
name: 'OTEL_TRACES_SAMPLER unset, OTEL_TRACES_SAMPLER_ARG=0 (no sampling)',
args: ['./fixtures/use-http-get.js'],
cwd: __dirname,
env: {
NODE_OPTIONS: '--import=@elastic/opentelemetry-node',
OTEL_TRACES_SAMPLER_ARG: '0',
},
checkTelemetry: (t, col) => {
t.equal(col.sortedSpans.length, 0);
},
},
{
name: 'OTEL_TRACES_SAMPLER=always_off, (no sampling)',
args: ['./fixtures/use-http-get.js'],
cwd: __dirname,
env: {
NODE_OPTIONS: '--import=@elastic/opentelemetry-node',
OTEL_TRACES_SAMPLER: 'always_off',
},
checkTelemetry: (t, col) => {
t.equal(col.sortedSpans.length, 0);
},
},
];

// ----- main line -----

test('OTEL_TRACES_SAMPLER', (suite) => {
runTestFixtures(suite, testFixtures);
suite.end();
});
Loading
Loading