feat(opentelemetry-configuration): parse of environment variables on configuration file (#5949)

Co-authored-by: Marc Pichler <[email protected]>
Co-authored-by: Trent Mick <[email protected]>

Author: 3 people

experimental/CHANGELOG.md

@@ -15,6 +15,7 @@ For notes on migrating to 2.x / 0.200.x see [the upgrade guide](doc/upgrade-to-2
 * feat(opentelemetry-configuration): add more attributes to config model [#5826](https://github.com/open-telemetry/opentelemetry-js/pull/5826) @maryliag
 * feat(opentelemetry-configuration): Parse of Configuration File [#5875](https://github.com/open-telemetry/opentelemetry-js/pull/5875) @maryliag
 * feat(opentelemetry-configuration): parse of array objects on configuration file [#5947](https://github.com/open-telemetry/opentelemetry-js/pull/5947) @maryliag
+* feat(opentelemetry-configuration): parse of environment variables on configuration file [#5947](https://github.com/open-telemetry/opentelemetry-js/pull/5947) @maryliag
 
 ### :bug: Bug Fixes
 

experimental/packages/opentelemetry-configuration/src/utils.ts

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 import { diag } from '@opentelemetry/api';
+import { getStringFromEnv } from '@opentelemetry/core';
 import { inspect } from 'util';
 
 /**
@@ -26,7 +27,7 @@ import { inspect } from 'util';
  * @returns {boolean} - The boolean value or `false` if the environment variable is unset empty, unset, or contains only whitespace.
  */
 export function getBooleanFromConfigFile(value: unknown): boolean | undefined {
-  const raw = String(value)?.trim().toLowerCase();
+  const raw = envVariableSubstitution(value)?.trim().toLowerCase();
   if (raw === 'true') {
     return true;
   } else if (raw === 'false') {
@@ -77,7 +78,7 @@ export function getBooleanListFromConfigFile(
  * @returns {number | undefined} - The number value or `undefined`.
  */
 export function getNumberFromConfigFile(value: unknown): number | undefined {
-  const raw = String(value)?.trim();
+  const raw = envVariableSubstitution(value)?.trim();
   if (raw == null || raw.trim() === '') {
     return undefined;
   }
@@ -127,7 +128,7 @@ export function getNumberListFromConfigFile(
  * @returns {string | undefined} - The string value or `undefined`.
  */
 export function getStringFromConfigFile(value: unknown): string | undefined {
-  const raw = String(value)?.trim();
+  const raw = envVariableSubstitution(value)?.trim();
   if (value == null || raw === '') {
     return undefined;
   }
@@ -148,8 +149,28 @@ export function getStringFromConfigFile(value: unknown): string | undefined {
 export function getStringListFromConfigFile(
   value: unknown
 ): string[] | undefined {
+  value = envVariableSubstitution(value);
   return getStringFromConfigFile(value)
     ?.split(',')
     .map(v => v.trim())
     .filter(s => s !== '');
 }
+
+export function envVariableSubstitution(value: unknown): string | undefined {
+  if (value == null) {
+    return undefined;
+  }
+
+  const matches = String(value).match(/\$\{[a-zA-Z0-9_:.-]*\}/g);
+  if (matches) {
+    let stringValue = String(value);
+    for (const match of matches) {
+      const v = match.substring(2, match.length - 1).split(':-');
+      const defaultValue = v.length === 2 ? v[1] : '';
+      const replacement = getStringFromEnv(v[0]) || defaultValue;
+      stringValue = stringValue.replace(match, replacement);
+    }
+    return stringValue;
+  }
+  return String(value);
+}

experimental/packages/opentelemetry-configuration/test/ConfigProvider.test.ts

@@ -245,6 +245,102 @@ const configFromFile: Configuration = {
   },
 };
 
+const defaultConfigFromFileWithEnvVariables: Configuration = {
+  disabled: false,
+  log_level: DiagLogLevel.INFO,
+  node_resource_detectors: ['all'],
+  resource: {
+    attributes: [
+      {
+        name: 'service.name',
+        value: 'unknown_service',
+        type: 'string',
+      },
+    ],
+  },
+  attribute_limits: {
+    attribute_count_limit: 128,
+  },
+  propagator: {
+    composite: ['tracecontext', 'baggage'],
+    composite_list: 'tracecontext,baggage',
+  },
+  tracer_provider: {
+    processors: [
+      {
+        batch: {
+          schedule_delay: 5000,
+          export_timeout: 30000,
+          max_queue_size: 2048,
+          max_export_batch_size: 512,
+          exporter: {
+            otlp_http: {
+              endpoint: 'http://localhost:4318/v1/traces',
+              timeout: 10000,
+            },
+          },
+        },
+      },
+    ],
+    limits: {
+      attribute_count_limit: 128,
+      event_count_limit: 128,
+      link_count_limit: 128,
+      event_attribute_count_limit: 128,
+      link_attribute_count_limit: 128,
+    },
+    sampler: {
+      parent_based: {
+        root: 'always_on',
+        remote_parent_sampled: 'always_on',
+        remote_parent_not_sampled: 'always_off',
+        local_parent_sampled: 'always_on',
+        local_parent_not_sampled: 'always_off',
+      },
+    },
+  },
+  meter_provider: {
+    readers: [
+      {
+        periodic: {
+          interval: 60000,
+          timeout: 30000,
+          exporter: {
+            otlp_http: {
+              endpoint: 'http://localhost:4318/v1/metrics',
+              timeout: 10000,
+              temporality_preference: 'cumulative',
+              default_histogram_aggregation: 'explicit_bucket_histogram',
+            },
+          },
+        },
+      },
+    ],
+    exemplar_filter: 'trace_based',
+  },
+  logger_provider: {
+    processors: [
+      {
+        batch: {
+          schedule_delay: 1000,
+          export_timeout: 30000,
+          max_queue_size: 2048,
+          max_export_batch_size: 512,
+          exporter: {
+            otlp_http: {
+              endpoint: 'http://localhost:4318/v1/logs',
+              timeout: 10000,
+            },
+          },
+        },
+      },
+    ],
+    limits: {
+      attribute_count_limit: 128,
+    },
+  },
+};
+
 describe('ConfigProvider', function () {
   describe('get values from environment variables', function () {
     afterEach(function () {
@@ -602,6 +698,56 @@ describe('ConfigProvider', function () {
     afterEach(function () {
       delete process.env.OTEL_EXPERIMENTAL_CONFIG_FILE;
       delete process.env.OTEL_NODE_RESOURCE_DETECTORS;
+      delete process.env.OTEL_SDK_DISABLED;
+      delete process.env.OTEL_LOG_LEVEL;
+      delete process.env.OTEL_SERVICE_NAME;
+      delete process.env.OTEL_RESOURCE_ATTRIBUTES;
+      delete process.env.OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT;
+      delete process.env.OTEL_ATTRIBUTE_COUNT_LIMIT;
+      delete process.env.OTEL_PROPAGATORS;
+      delete process.env.OTEL_BSP_SCHEDULE_DELAY;
+      delete process.env.OTEL_BSP_EXPORT_TIMEOUT;
+      delete process.env.OTEL_BSP_MAX_QUEUE_SIZE;
+      delete process.env.OTEL_BSP_MAX_EXPORT_BATCH_SIZE;
+      delete process.env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT;
+      delete process.env.OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE;
+      delete process.env.OTEL_EXPORTER_OTLP_TRACES_CLIENT_KEY;
+      delete process.env.OTEL_EXPORTER_OTLP_TRACES_CLIENT_CERTIFICATE;
+      delete process.env.OTEL_EXPORTER_OTLP_TRACES_COMPRESSION;
+      delete process.env.OTEL_EXPORTER_OTLP_TRACES_TIMEOUT;
+      delete process.env.OTEL_EXPORTER_OTLP_TRACES_HEADERS;
+      delete process.env.OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT;
+      delete process.env.OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT;
+      delete process.env.OTEL_SPAN_EVENT_COUNT_LIMIT;
+      delete process.env.OTEL_SPAN_LINK_COUNT_LIMIT;
+      delete process.env.OTEL_EVENT_ATTRIBUTE_COUNT_LIMIT;
+      delete process.env.OTEL_LINK_ATTRIBUTE_COUNT_LIMIT;
+      delete process.env.OTEL_METRIC_EXPORT_INTERVAL;
+      delete process.env.OTEL_METRIC_EXPORT_TIMEOUT;
+      delete process.env.OTEL_EXPORTER_OTLP_METRICS_ENDPOINT;
+      delete process.env.OTEL_EXPORTER_OTLP_METRICS_CERTIFICATE;
+      delete process.env.OTEL_EXPORTER_OTLP_METRICS_CLIENT_KEY;
+      delete process.env.OTEL_EXPORTER_OTLP_METRICS_CLIENT_CERTIFICATE;
+      delete process.env.OTEL_EXPORTER_OTLP_METRICS_COMPRESSION;
+      delete process.env.OTEL_EXPORTER_OTLP_METRICS_TIMEOUT;
+      delete process.env.OTEL_EXPORTER_OTLP_METRICS_HEADERS;
+      delete process.env.OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE;
+      delete process.env
+        .OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION;
+      delete process.env.OTEL_METRICS_EXEMPLAR_FILTER;
+      delete process.env.OTEL_BLRP_SCHEDULE_DELAY;
+      delete process.env.OTEL_BLRP_EXPORT_TIMEOUT;
+      delete process.env.OTEL_BLRP_MAX_QUEUE_SIZE;
+      delete process.env.OTEL_BLRP_MAX_EXPORT_BATCH_SIZE;
+      delete process.env.OTEL_EXPORTER_OTLP_LOGS_ENDPOINT;
+      delete process.env.OTEL_EXPORTER_OTLP_LOGS_CERTIFICATE;
+      delete process.env.OTEL_EXPORTER_OTLP_LOGS_CLIENT_KEY;
+      delete process.env.OTEL_EXPORTER_OTLP_LOGS_CLIENT_CERTIFICATE;
+      delete process.env.OTEL_EXPORTER_OTLP_LOGS_COMPRESSION;
+      delete process.env.OTEL_EXPORTER_OTLP_LOGS_TIMEOUT;
+      delete process.env.OTEL_EXPORTER_OTLP_LOGS_HEADERS;
+      delete process.env.OTEL_LOGRECORD_ATTRIBUTE_VALUE_LENGTH_LIMIT;
+      delete process.env.OTEL_LOGRECORD_ATTRIBUTE_COUNT_LIMIT;
     });
 
     it('should initialize config with default values from valid config file', function () {
@@ -655,5 +801,94 @@ describe('ConfigProvider', function () {
         defaultConfig
       );
     });
+
+    it('should initialize config with config file that contains environment variables', function () {
+      process.env.OTEL_EXPERIMENTAL_CONFIG_FILE =
+        'test/fixtures/sdk-migration-config.yaml';
+      process.env.OTEL_SDK_DISABLED = 'false';
+      process.env.OTEL_LOG_LEVEL = 'debug';
+      process.env.OTEL_SERVICE_NAME = 'custom-name';
+      process.env.OTEL_RESOURCE_ATTRIBUTES = 'attributes';
+      process.env.OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT = '23';
+      process.env.OTEL_ATTRIBUTE_COUNT_LIMIT = '7';
+      process.env.OTEL_PROPAGATORS = 'prop';
+      process.env.OTEL_BSP_SCHEDULE_DELAY = '123';
+      process.env.OTEL_BSP_EXPORT_TIMEOUT = '456';
+      process.env.OTEL_BSP_MAX_QUEUE_SIZE = '789';
+      process.env.OTEL_BSP_MAX_EXPORT_BATCH_SIZE = '1011';
+      process.env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT = 'trace-endpoint';
+      process.env.OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE = 'trace-certificate';
+      process.env.OTEL_EXPORTER_OTLP_TRACES_CLIENT_KEY = 'trace-client-key';
+      process.env.OTEL_EXPORTER_OTLP_TRACES_CLIENT_CERTIFICATE =
+        'trace-client-certificate';
+      process.env.OTEL_EXPORTER_OTLP_TRACES_COMPRESSION = 'trace-compression';
+      process.env.OTEL_EXPORTER_OTLP_TRACES_TIMEOUT = '1213';
+      process.env.OTEL_EXPORTER_OTLP_TRACES_HEADERS = 'trace-headers';
+      process.env.OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT = '14';
+      process.env.OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT = '15';
+      process.env.OTEL_SPAN_EVENT_COUNT_LIMIT = '16';
+      process.env.OTEL_SPAN_LINK_COUNT_LIMIT = '17';
+      process.env.OTEL_EVENT_ATTRIBUTE_COUNT_LIMIT = '18';
+      process.env.OTEL_LINK_ATTRIBUTE_COUNT_LIMIT = '19';
+      process.env.OTEL_METRIC_EXPORT_INTERVAL = '20';
+      process.env.OTEL_METRIC_EXPORT_TIMEOUT = '21';
+      process.env.OTEL_EXPORTER_OTLP_METRICS_ENDPOINT = 'metric-endpoint';
+      process.env.OTEL_EXPORTER_OTLP_METRICS_CERTIFICATE = 'metric-certificate';
+      process.env.OTEL_EXPORTER_OTLP_METRICS_CLIENT_KEY = 'metric-client-key';
+      process.env.OTEL_EXPORTER_OTLP_METRICS_CLIENT_CERTIFICATE =
+        'metric-client-certificate';
+      process.env.OTEL_EXPORTER_OTLP_METRICS_COMPRESSION = 'metric-compression';
+      process.env.OTEL_EXPORTER_OTLP_METRICS_TIMEOUT = '22';
+      process.env.OTEL_EXPORTER_OTLP_METRICS_HEADERS = 'metric-header';
+      process.env.OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE =
+        'metric-temporality';
+      process.env.OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION =
+        'metric-hist-agg';
+      process.env.OTEL_METRICS_EXEMPLAR_FILTER = 'metric-exemplar-filter';
+      process.env.OTEL_BLRP_SCHEDULE_DELAY = '23';
+      process.env.OTEL_BLRP_EXPORT_TIMEOUT = '24';
+      process.env.OTEL_BLRP_MAX_QUEUE_SIZE = '25';
+      process.env.OTEL_BLRP_MAX_EXPORT_BATCH_SIZE = '26';
+      process.env.OTEL_EXPORTER_OTLP_LOGS_ENDPOINT = 'logs-endpoint';
+      process.env.OTEL_EXPORTER_OTLP_LOGS_CERTIFICATE = 'logs-certificate';
+      process.env.OTEL_EXPORTER_OTLP_LOGS_CLIENT_KEY = 'logs-client-key';
+      process.env.OTEL_EXPORTER_OTLP_LOGS_CLIENT_CERTIFICATE =
+        'logs-client-certificate';
+      process.env.OTEL_EXPORTER_OTLP_LOGS_COMPRESSION = 'logs-compression';
+      process.env.OTEL_EXPORTER_OTLP_LOGS_TIMEOUT = '27';
+      process.env.OTEL_EXPORTER_OTLP_LOGS_HEADERS = 'logs-header';
+      process.env.OTEL_LOGRECORD_ATTRIBUTE_VALUE_LENGTH_LIMIT = '28';
+      process.env.OTEL_LOGRECORD_ATTRIBUTE_COUNT_LIMIT = '29';
+      const configProvider = createConfigProvider();
+      const expectedConfig: Configuration = {
+        ...defaultConfig,
+        resource: {
+          attributes_list: 'attributes',
+          attributes: [
+            {
+              name: 'service.name',
+              value: 'custom-name',
+              type: 'string',
+            },
+          ],
+        },
+      };
+
+      assert.deepStrictEqual(
+        configProvider.getInstrumentationConfig(),
+        expectedConfig
+      );
+    });
+
+    it('should initialize config with fallbacks defined in config file when corresponding environment variables are not defined', function () {
+      process.env.OTEL_EXPERIMENTAL_CONFIG_FILE =
+        'test/fixtures/sdk-migration-config.yaml';
+
+      const configProvider = createConfigProvider();
+      assert.deepStrictEqual(
+        configProvider.getInstrumentationConfig(),
+        defaultConfigFromFileWithEnvVariables
+      );
+    });
   });
 });