From 58a54ced14797be76013f88e04df38bea0cdc4bd Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Thu, 26 Jun 2025 06:02:26 -0400 Subject: [PATCH 01/15] begin support for otel declarative configuration --- apps/opentelemetry/src/otel_configuration.erl | 41 ++- config/sdk-config.yaml | 270 ++++++++++++++++++ config/sdk.config | 105 +++++++ 3 files changed, 411 insertions(+), 5 deletions(-) create mode 100644 config/sdk-config.yaml create mode 100644 config/sdk.config diff --git a/apps/opentelemetry/src/otel_configuration.erl b/apps/opentelemetry/src/otel_configuration.erl index 2b652dd6..6e53a9c9 100644 --- a/apps/opentelemetry/src/otel_configuration.erl +++ b/apps/opentelemetry/src/otel_configuration.erl @@ -113,13 +113,44 @@ new() -> merge_with_os(AppEnv) -> ConfigMap = new(), + ConfigMap1 = lists:foldl(fun(F, Acc) -> + F(AppEnv, Acc) + end, ConfigMap, [fun resource/2, + fun attribute_limits/2, + fun propagator/2, + fun tracer_provider/2, + fun sweeper/2]), + + %% backwards compatability lists:foldl(fun(F, Acc) -> F(AppEnv, Acc) - end, ConfigMap, [fun span_limits/2, - fun general/2, - fun sampler/2, - fun processors/2, - fun sweeper/2]). + end, ConfigMap1, [fun span_limits/2, + fun general/2, + fun sampler/2, + fun processors/2, + fun sweeper/2]). + +-spec resource(list(), t()) -> t(). +resource(AppEnv, ConfigMap) -> + Resource = proplists:get_value(resource, AppEnv, []), + ConfigMap. + +-spec attribute_limits(list(), t()) -> t(). +attribute_limits(AppEnv, ConfigMap) -> + AttributeLimits = proplists:get_value(attribute_limits, AppEnv, []), + ConfigMap. + +-spec propagator(list(), t()) -> t(). +propagator(AppEnv, ConfigMap) -> + Propagator = proplists:get_value(propagator, AppEnv, []), + ConfigMap. + +-spec tracer_provider(list(), t()) -> t(). +tracer_provider(AppEnv, ConfigMap) -> + TracerProvider = proplists:get_value(tracer_provider, AppEnv, []), + ConfigMap. + +%% backwards compatability -spec span_limits(list(), t()) -> t(). span_limits(AppEnv, ConfigMap) -> diff --git a/config/sdk-config.yaml b/config/sdk-config.yaml new file mode 100644 index 00000000..224937af --- /dev/null +++ b/config/sdk-config.yaml @@ -0,0 +1,270 @@ +# sdk-config.yaml is a typical starting point for configuring the SDK, including exporting to +# localhost via OTLP. + +# NOTE: With the exception of env var substitution syntax (i.e. ${MY_ENV}), SDKs ignore +# environment variables when interpreting config files. This including ignoring all env +# vars defined in https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/. + +# The file format version. +# The yaml format is documented at +# https://github.com/open-telemetry/opentelemetry-configuration/tree/main/schema +file_format: "1.0-rc.1" +# Configure if the SDK is disabled or not. +# If omitted or null, false is used. +disabled: false +# Configure the log level of the internal logger used by the SDK. +# If omitted, info is used. +log_level: info +# Configure resource for all signals. +# If omitted, the default resource is used. +resource: + # Configure resource attributes. Entries have higher priority than entries from .resource.attributes_list. + # Entries must contain .name and .value, and may optionally include .type. If an entry's .type omitted or null, string is used. + # The .value's type must match the .type. Values for .type include: string, bool, int, double, string_array, bool_array, int_array, double_array. + attributes: + - name: service.name + value: unknown_service +# Configure general attribute limits. See also tracer_provider.limits, logger_provider.limits. +attribute_limits: + # Configure max attribute value size. + # Value must be non-negative. + # If omitted or null, there is no limit. + attribute_value_length_limit: + # Configure max attribute count. + # Value must be non-negative. + # If omitted or null, 128 is used. + attribute_count_limit: 128 +# Configure text map context propagators. +# If omitted, a noop propagator is used. +propagator: + # Configure the propagators in the composite text map propagator. Entries from .composite_list are appended to the list here with duplicates filtered out. + # Built-in propagator keys include: tracecontext, baggage, b3, b3multi, jaeger, ottrace. Known third party keys include: xray. + # If the resolved list of propagators (from .composite and .composite_list) is empty, a noop propagator is used. + composite: + - # Include the w3c trace context propagator. + tracecontext: + - # Include the w3c baggage propagator. + baggage: +# Configure tracer provider. +# If omitted, a noop tracer provider is used. +tracer_provider: + # Configure span processors. + processors: + - # Configure a batch span processor. + batch: + # Configure delay interval (in milliseconds) between two consecutive exports. + # Value must be non-negative. + # If omitted or null, 5000 is used. + schedule_delay: 5000 + # Configure maximum allowed time (in milliseconds) to export data. + # Value must be non-negative. A value of 0 indicates no limit (infinity). + # If omitted or null, 30000 is used. + export_timeout: 30000 + # Configure maximum queue size. Value must be positive. + # If omitted or null, 2048 is used. + max_queue_size: 2048 + # Configure maximum batch size. Value must be positive. + # If omitted or null, 512 is used. + max_export_batch_size: 512 + # Configure exporter. + exporter: + # Configure exporter to be OTLP with HTTP transport. + otlp_http: + # Configure endpoint, including the trace specific path. + # If omitted or null, http://localhost:4318/v1/traces is used. + endpoint: http://localhost:4318/v1/traces + # Configure certificate used to verify a server's TLS credentials. + # Absolute path to certificate file in PEM format. + # If omitted or null, system default certificate verification is used for secure connections. + certificate_file: + # Configure mTLS private client key. + # Absolute path to client key file in PEM format. If set, .client_certificate must also be set. + # If omitted or null, mTLS is not used. + client_key_file: + # Configure mTLS client certificate. + # Absolute path to client certificate file in PEM format. If set, .client_key must also be set. + # If omitted or null, mTLS is not used. + client_certificate_file: + # Configure compression. + # Values include: gzip, none. Implementations may support other compression algorithms. + # If omitted or null, none is used. + compression: gzip + # Configure max time (in milliseconds) to wait for each export. + # Value must be non-negative. A value of 0 indicates no limit (infinity). + # If omitted or null, 10000 is used. + timeout: 10000 + # Configure headers. Entries have higher priority than entries from .headers_list. + # If an entry's .value is null, the entry is ignored. + headers: [] + # Configure span limits. See also attribute_limits. + limits: + # Configure max attribute value size. Overrides .attribute_limits.attribute_value_length_limit. + # Value must be non-negative. + # If omitted or null, there is no limit. + attribute_value_length_limit: + # Configure max attribute count. Overrides .attribute_limits.attribute_count_limit. + # Value must be non-negative. + # If omitted or null, 128 is used. + attribute_count_limit: 128 + # Configure max span event count. + # Value must be non-negative. + # If omitted or null, 128 is used. + event_count_limit: 128 + # Configure max span link count. + # Value must be non-negative. + # If omitted or null, 128 is used. + link_count_limit: 128 + # Configure max attributes per span event. + # Value must be non-negative. + # If omitted or null, 128 is used. + event_attribute_count_limit: 128 + # Configure max attributes per span link. + # Value must be non-negative. + # If omitted or null, 128 is used. + link_attribute_count_limit: 128 + # Configure the sampler. + # If omitted, parent based sampler with a root of always_on is used. + sampler: + # Configure sampler to be parent_based. + parent_based: + # Configure root sampler. + # If omitted or null, always_on is used. + root: + # Configure sampler to be always_on. + always_on: + # Configure remote_parent_sampled sampler. + # If omitted or null, always_on is used. + remote_parent_sampled: + # Configure sampler to be always_on. + always_on: + # Configure remote_parent_not_sampled sampler. + # If omitted or null, always_off is used. + remote_parent_not_sampled: + # Configure sampler to be always_off. + always_off: + # Configure local_parent_sampled sampler. + # If omitted or null, always_on is used. + local_parent_sampled: + # Configure sampler to be always_on. + always_on: + # Configure local_parent_not_sampled sampler. + # If omitted or null, always_off is used. + local_parent_not_sampled: + # Configure sampler to be always_off. + always_off: +# Configure meter provider. +# If omitted, a noop meter provider is used. +meter_provider: + # Configure metric readers. + readers: + - # Configure a periodic metric reader. + periodic: + # Configure delay interval (in milliseconds) between start of two consecutive exports. + # Value must be non-negative. + # If omitted or null, 60000 is used. + interval: 60000 + # Configure maximum allowed time (in milliseconds) to export data. + # Value must be non-negative. A value of 0 indicates no limit (infinity). + # If omitted or null, 30000 is used. + timeout: 30000 + # Configure exporter. + exporter: + # Configure exporter to be OTLP with HTTP transport. + otlp_http: + # Configure endpoint, including the metric specific path. + # If omitted or null, http://localhost:4318/v1/metrics is used. + endpoint: http://localhost:4318/v1/metrics + # Configure certificate used to verify a server's TLS credentials. + # Absolute path to certificate file in PEM format. + # If omitted or null, system default certificate verification is used for secure connections. + certificate_file: + # Configure mTLS private client key. + # Absolute path to client key file in PEM format. If set, .client_certificate must also be set. + # If omitted or null, mTLS is not used. + client_key_file: + # Configure mTLS client certificate. + # Absolute path to client certificate file in PEM format. If set, .client_key must also be set. + # If omitted or null, mTLS is not used. + client_certificate_file: + # Configure compression. + # Values include: gzip, none. Implementations may support other compression algorithms. + # If omitted or null, none is used. + compression: gzip + # Configure max time (in milliseconds) to wait for each export. + # Value must be non-negative. A value of 0 indicates no limit (infinity). + # If omitted or null, 10000 is used. + timeout: 10000 + # Configure headers. Entries have higher priority than entries from .headers_list. + # If an entry's .value is null, the entry is ignored. + headers: [] + # Configure temporality preference. + # Values include: cumulative, delta, low_memory. For behavior of values, see https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/sdk_exporters/otlp.md. + # If omitted or null, cumulative is used. + temporality_preference: cumulative + # Configure default histogram aggregation. + # Values include: explicit_bucket_histogram, base2_exponential_bucket_histogram. For behavior of values, see https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/sdk_exporters/otlp.md. + # If omitted or null, explicit_bucket_histogram is used. + default_histogram_aggregation: explicit_bucket_histogram + # Configure the exemplar filter. + # Values include: trace_based, always_on, always_off. For behavior of values see https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/configuration/sdk-environment-variables.md#metrics-sdk-configuration. + # If omitted or null, trace_based is used. + exemplar_filter: trace_based +# Configure logger provider. +# If omitted, a noop logger provider is used. +logger_provider: + # Configure log record processors. + processors: + - # Configure a batch log record processor. + batch: + # Configure delay interval (in milliseconds) between two consecutive exports. + # Value must be non-negative. + # If omitted or null, 1000 is used. + schedule_delay: 1000 + # Configure maximum allowed time (in milliseconds) to export data. + # Value must be non-negative. A value of 0 indicates no limit (infinity). + # If omitted or null, 30000 is used. + export_timeout: 30000 + # Configure maximum queue size. Value must be positive. + # If omitted or null, 2048 is used. + max_queue_size: 2048 + # Configure maximum batch size. Value must be positive. + # If omitted or null, 512 is used. + max_export_batch_size: 512 + # Configure exporter. + exporter: + # Configure exporter to be OTLP with HTTP transport. + otlp_http: + endpoint: http://localhost:4318/v1/logs + # Configure certificate used to verify a server's TLS credentials. + # Absolute path to certificate file in PEM format. + # If omitted or null, system default certificate verification is used for secure connections. + certificate_file: + # Configure mTLS private client key. + # Absolute path to client key file in PEM format. If set, .client_certificate must also be set. + # If omitted or null, mTLS is not used. + client_key_file: + # Configure mTLS client certificate. + # Absolute path to client certificate file in PEM format. If set, .client_key must also be set. + # If omitted or null, mTLS is not used. + client_certificate_file: + # Configure compression. + # Values include: gzip, none. Implementations may support other compression algorithms. + # If omitted or null, none is used. + compression: gzip + # Configure max time (in milliseconds) to wait for each export. + # Value must be non-negative. A value of 0 indicates no limit (infinity). + # If omitted or null, 10000 is used. + timeout: 10000 + # Configure headers. Entries have higher priority than entries from .headers_list. + # If an entry's .value is null, the entry is ignored. + headers: [] + # Configure log record limits. See also attribute_limits. + limits: + # Configure max attribute value size. Overrides .attribute_limits.attribute_value_length_limit. + # Value must be non-negative. + # If omitted or null, there is no limit. + attribute_value_length_limit: + # Configure max attribute count. Overrides .attribute_limits.attribute_count_limit. + # Value must be non-negative. + # If omitted or null, 128 is used. + attribute_count_limit: 128 diff --git a/config/sdk.config b/config/sdk.config new file mode 100644 index 00000000..e9febbb6 --- /dev/null +++ b/config/sdk.config @@ -0,0 +1,105 @@ +[ + {file_format, "1.0-rc.1"}, + {disabled, false}, + {log_level, info}, + {resource, [ + {attributes, [ + #{name => <<"service.name">>, value => <<"unknown_service">>} + ]} + ]}, + {attribute_limits, [ + {attribute_value_length_limit, undefined}, + {attribute_count_limit, 128} + ]}, + {propagator, [ + {composite, [tracecontext, baggage]} + ]}, + + {tracer_provider, [ + {processors, [ + {batch, [ + {schedule_delay, 5000}, + {export_timeout, 30000}, + {max_queue_size, 2048}, + {max_export_batch_size, 512}, + {exporter, [ + {otlp_http, [ + {endpoint, "http://localhost:4318/v1/traces"}, + {certificate_file, undefined}, + {client_key_file, undefined}, + {client_certificate_file, undefined}, + {compression, gzip}, + {timeout, 10000}, + {headers, []} + ]} + ]} + ]} + ]}, + {limits, [ + {attribute_value_length_limit, undefined}, + {attribute_count_limit, 128}, + {event_count_limit, 128}, + {link_count_limit, 128}, + {event_attribute_count_limit, 128}, + {link_attribute_count_limit, 128} + ]}, + {sampler, [ + {parent_based, [ + {root, always_on}, + {remote_parent_sampled, always_on}, + {remote_parent_not_sampled, always_off}, + {local_parent_sampled, always_on}, + {local_parent_not_sampled, always_off} + ]} + ]} + ]}, + + {meter_provider, [ + {readers, [ + {periodic, [ + {interval, 60000}, + {timeout, 30000}, + {exporter, [ + {otlp_http, [ + {endpoint, "http://localhost:4318/v1/metrics"}, + {certificate_file, undefined}, + {client_key_file, undefined}, + {client_certificate_file, undefined}, + {compression, gzip}, + {timeout, 10000}, + {headers, []}, + {temporality_preference, cumulative}, + {default_histogram_aggregation, explicit_bucket_histogram} + ]} + ]} + ]} + ]}, + {exemplar_filter, trace_based} + ]}, + + {logger_provider, [ + {processors, [ + {batch, [ + {schedule_delay, 1000}, + {export_timeout, 30000}, + {max_queue_size, 2048}, + {max_export_batch_size, 512}, + {exporter, [ + {otlp_http, [ + {endpoint, "http://localhost:4318/v1/logs"}, + {certificate_file, undefined}, + {client_key_file, undefined}, + {client_certificate_file, undefined}, + {compression, gzip}, + {timeout, 10000}, + {headers, []} + ]} + ]} + ]} + ]}, + {limits, [ + {attribute_value_length_limit, undefined}, + {attribute_count_limit, 128} + ]} + ]} +]. From 3d2197481a83fb61da494ae5c717c4d723593eaa Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Thu, 3 Jul 2025 05:48:32 -0400 Subject: [PATCH 02/15] use new sdk configuration and move defaults sdk_disabled flag from the old flat structure is converted to new format from configuraiton SIG. defaults are read where the configuration is read. this is so a component can know the difference between a configuration key being defined as empty and being undefined. before the default was set for the key when configuration was read, meaning components could not distinguish between if the configuration was empty, undefined or set to the default by the user. --- CHANGELOG.md | 8 + apps/opentelemetry/src/opentelemetry_app.erl | 5 +- apps/opentelemetry/src/opentelemetry_sup.erl | 2 +- apps/opentelemetry/src/otel_configuration.erl | 270 +++++++++++------- .../src/otel_resource_detector.erl | 8 +- apps/opentelemetry/src/otel_span_limits.erl | 13 +- apps/opentelemetry/src/otel_span_sup.erl | 2 +- apps/opentelemetry/src/otel_span_sweeper.erl | 9 +- apps/opentelemetry/src/otel_tracer_server.erl | 10 +- .../test/otel_configuration_SUITE.erl | 18 +- .../include/opentelemetry.hrl | 2 + config/sdk.config | 18 +- 12 files changed, 213 insertions(+), 152 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff619eac..f4650775 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### SDK + +### + +- []() + +### Changes + ## Experimental API 0.5.2 - 2024-11-22 ### Added diff --git a/apps/opentelemetry/src/opentelemetry_app.erl b/apps/opentelemetry/src/opentelemetry_app.erl index a24f836b..7b6151c1 100644 --- a/apps/opentelemetry/src/opentelemetry_app.erl +++ b/apps/opentelemetry/src/opentelemetry_app.erl @@ -34,7 +34,7 @@ start(_StartType, _StartArgs) -> SupResult = opentelemetry_sup:start_link(Config), case Config of - #{sdk_disabled := true} -> + #{disabled := true} -> %% skip the rest if the SDK is disabled SupResult; _ -> @@ -57,7 +57,8 @@ stop(_State) -> %% internal functions -setup_text_map_propagators(#{text_map_propagators := List}) -> +setup_text_map_propagators(Config) -> + List = maps:get(text_map_propagators, Config, []), CompositePropagator = otel_propagator_text_map_composite:create(List), opentelemetry:set_text_map_propagator(CompositePropagator). diff --git a/apps/opentelemetry/src/opentelemetry_sup.erl b/apps/opentelemetry/src/opentelemetry_sup.erl index cb0ff0b5..9c022917 100644 --- a/apps/opentelemetry/src/opentelemetry_sup.erl +++ b/apps/opentelemetry/src/opentelemetry_sup.erl @@ -29,7 +29,7 @@ start_link(Opts) -> supervisor:start_link({local, ?SERVER}, ?MODULE, [Opts]). -init([#{sdk_disabled := true}]) -> +init([#{disabled := true}]) -> {ok, {#{}, []}}; init([Opts]) -> SupFlags = #{strategy => one_for_one, diff --git a/apps/opentelemetry/src/otel_configuration.erl b/apps/opentelemetry/src/otel_configuration.erl index 6e53a9c9..f34a74f4 100644 --- a/apps/opentelemetry/src/otel_configuration.erl +++ b/apps/opentelemetry/src/otel_configuration.erl @@ -23,50 +23,85 @@ merge_list_with_environment/3, report_cb/1]). --define(BATCH_PROCESSOR_DEFAULTS, #{scheduled_delay_ms => 5000, - exporting_timeout_ms => 30000, - max_queue_size => 2048, - exporter => {opentelemetry_exporter, #{}}}). --define(SIMPLE_PROCESSOR_DEFAULTS, #{exporting_timeout_ms => 30000, - exporter => {opentelemetry_exporter, #{}}}). - -%% required configuration -%% using a map instead of a record because there can be more values --type t() :: #{sdk_disabled := boolean(), - log_level := atom(), - register_loaded_applications := boolean() | undefined, - create_application_tracers := boolean() | undefined, - id_generator := module(), - deny_list := [atom()], - - resource_detectors := [module()], - resource_detector_timeout := integer(), - bsp_scheduled_delay_ms := integer() | undefined, - bsp_exporting_timeout_ms := integer() | undefined, - bsp_max_queue_size := integer() | undefined, - ssp_exporting_timeout_ms := integer() | undefined, - text_map_propagators := [atom()], - traces_exporter := {atom(), term()} | none | undefined, - metrics_exporter := {atom(), term()} | none | undefined, - views := list(), %% TODO: type should be `[otel_meter_server:view_config]' - %% when Metrics are moved out of the experimental app - readers := [#{id := atom(), module => module(), config => map()}], - exemplars_enabled := boolean(), - exemplar_filter := always_on | always_off | trace_based, - metric_producers := [{module(), term()}], - - processors := list(), - sampler := {atom(), term()}, - sweeper := #{interval => integer() | infinity, - strategy => atom() | fun(), - span_ttl => integer() | infinity, - storage_size => integer() | infinity}, - attribute_count_limit := integer(), - attribute_value_length_limit := integer() | infinity, - event_count_limit := integer(), - link_count_limit := integer(), - attribute_per_event_limit := integer(), - attribute_per_link_limit := integer()}. +-type log_level() :: atom(). + +-type attribute_limits() :: #{attribute_value_length_limit := integer() | undefined, + attribute_count_limit := integer() | undefined + }. + +-type exporter_args() :: map(). + +-type exporter() :: {module(), exporter_args()}. + +-type batch_span_processor() :: #{schedule_delay := integer(), + export_timeout := integer(), + max_queue_size := integer(), + max_export_batch_size := integer(), + exporter := exporter()}. + +-type simple_span_processor() :: #{exporter := exporter()}. + +-type span_processor_type() :: batch | simple | atom(). + +-type span_processor() :: #{span_processor_type() => batch_span_processor() | simple_span_processor() | otel_config_properties:t()}. + +-type propagator() :: tracecontext | baggage | b3 | b3multi | jaeger | ottrace | atom(). + +-type t() :: #{disabled => boolean(), + log_level => log_level(), + resource => #{attributes => opentelemetry:attributes_map()}, + attribute_limits => attribute_limits(), + propagator := #{composite => [propagator()], + composite_list => string()}, + tracer_provider => #{processors => [span_processor()], + limits => #{}, + sampler => #{}} + }. + +%% structure before the standard otel declarative configuration +%% -type old_t() :: #{sdk_disabled := boolean(), +%% log_level := atom(), +%% register_loaded_applications := boolean() | undefined, +%% create_application_tracers := boolean() | undefined, +%% id_generator := module(), +%% deny_list := [atom()], + +%% resource_detectors := [module()], +%% resource_detector_timeout := integer(), + +%% attribute_count_limit := integer(), +%% attribute_value_length_limit := integer() | infinity, + +%% event_count_limit := integer(), +%% link_count_limit := integer(), + +%% attribute_per_event_limit := integer(), +%% attribute_per_link_limit := integer(), + +%% %% tracer provider +%% bsp_scheduled_delay_ms := integer() | undefined, +%% bsp_exporting_timeout_ms := integer() | undefined, +%% bsp_max_queue_size := integer() | undefined, +%% ssp_exporting_timeout_ms := integer() | undefined, +%% traces_exporter := {atom(), term()} | none | undefined, +%% processors := list(), +%% sampler := {atom(), term()}, + +%% text_map_propagators := [atom()], + +%% metrics_exporter := {atom(), term()} | none | undefined, +%% views := list(), %% TODO: type should be `[otel_meter_server:view_config]' +%% %% when Metrics are moved out of the experimental app +%% readers := [#{id := atom(), module => module(), config => map()}], +%% exemplars_enabled := boolean(), +%% exemplar_filter := always_on | always_off | trace_based, +%% metric_producers := [{module(), term()}], + +%% sweeper := #{interval => integer() | infinity, +%% strategy => atom() | fun(), +%% span_ttl => integer() | infinity, +%% storage_size => integer() | infinity} +%% }. -export_type([t/0]). @@ -75,60 +110,80 @@ -spec new() -> t(). new() -> - ?assert_type(#{sdk_disabled => false, - log_level => info, - register_loaded_applications => undefined, - create_application_tracers => undefined, - id_generator => otel_id_generator, - deny_list => [], - resource_detectors => [otel_resource_env_var, - otel_resource_app_env], - resource_detector_timeout => 5000, - bsp_scheduled_delay_ms => undefined, - bsp_exporting_timeout_ms => undefined, - bsp_max_queue_size => undefined, - ssp_exporting_timeout_ms => undefined, - text_map_propagators => [trace_context, baggage], - traces_exporter => {opentelemetry_exporter, #{}}, - metrics_exporter => {opentelemetry_exporter, #{}}, - views => [], - readers => [], - exemplars_enabled => false, - exemplar_filter => trace_based, - metric_producers => [], - processors => [{otel_batch_processor, ?BATCH_PROCESSOR_DEFAULTS}], - sampler => {parent_based, #{root => always_on}}, - sweeper => #{interval => timer:minutes(10), - strategy => drop, - span_ttl => timer:minutes(30), - storage_size => infinity}, - attribute_count_limit => 128, - attribute_value_length_limit => infinity, - event_count_limit => 128, - link_count_limit => 128, - attribute_per_event_limit => 128, - attribute_per_link_limit => 128}, t()). - + ?assert_type(#{}, t()). + +%% old() -> +%% ?assert_type(#{sdk_disabled => false, +%% log_level => info, +%% register_loaded_applications => undefined, +%% create_application_tracers => undefined, +%% id_generator => otel_id_generator, +%% deny_list => [], +%% resource_detectors => [otel_resource_env_var, +%% otel_resource_app_env], +%% resource_detector_timeout => 5000, +%% bsp_scheduled_delay_ms => undefined, +%% bsp_exporting_timeout_ms => undefined, +%% bsp_max_queue_size => undefined, +%% ssp_exporting_timeout_ms => undefined, +%% text_map_propagators => [trace_context, baggage], +%% traces_exporter => {opentelemetry_exporter, #{}}, +%% metrics_exporter => {opentelemetry_exporter, #{}}, +%% views => [], +%% readers => [], +%% exemplars_enabled => false, +%% exemplar_filter => trace_based, +%% metric_producers => [], +%% processors => [{otel_batch_processor, ?BATCH_PROCESSOR_DEFAULTS}], +%% sampler => {parent_based, #{root => always_on}}, +%% sweeper => #{interval => timer:minutes(10), +%% strategy => drop, +%% span_ttl => timer:minutes(30), +%% storage_size => infinity}, +%% attribute_count_limit => 128, +%% attribute_value_length_limit => infinity, +%% event_count_limit => 128, +%% link_count_limit => 128, +%% attribute_per_event_limit => 128, +%% attribute_per_link_limit => 128}, old_t()). + +%% constructs a config based on the flat format and merges with the new +%% nested configuration from the declarative configuration SIG -spec merge_with_os(list()) -> t(). merge_with_os(AppEnv) -> - ConfigMap = new(), - - ConfigMap1 = lists:foldl(fun(F, Acc) -> - F(AppEnv, Acc) - end, ConfigMap, [fun resource/2, - fun attribute_limits/2, - fun propagator/2, - fun tracer_provider/2, - fun sweeper/2]), - - %% backwards compatability - lists:foldl(fun(F, Acc) -> - F(AppEnv, Acc) - end, ConfigMap1, [fun span_limits/2, - fun general/2, - fun sampler/2, - fun processors/2, - fun sweeper/2]). + OldConfig = lists:foldl(fun(F, Acc) -> + F(AppEnv, Acc) + end, #{}, [fun span_limits/2, + fun general/2, + fun sampler/2, + fun processors/2, + fun sweeper/2]), + + convert_to_new(OldConfig). + +-spec convert_to_new(#{}) -> t(). +convert_to_new(OldConfig) -> + convert_disabled(OldConfig). + +convert_disabled(OldConfig) -> + case maps:take(sdk_disabled, OldConfig) of + {Value, OldConfig1} -> + OldConfig1#{disabled => Value}; + error -> + OldConfig + end. + + %% ConfigMap = new(), + + %% ConfigMap1 = lists:foldl(fun(F, Acc) -> + %% F(AppEnv, Acc) + %% end, ConfigMap, [fun resource/2, + %% fun attribute_limits/2, + %% fun propagator/2, + %% fun tracer_provider/2, + %% fun sweeper/2]), + + -spec resource(list(), t()) -> t(). resource(AppEnv, ConfigMap) -> @@ -167,7 +222,7 @@ general(AppEnv, ConfigMap) -> %% `create_application_tracers' isn't set so update %% with the `register_loaded_applications' value %% or `true' if it too isn't set - case maps:get(register_loaded_applications, Config) of + case maps:get(register_loaded_applications, Config, undefined) of undefined -> true; Bool -> @@ -175,12 +230,13 @@ general(AppEnv, ConfigMap) -> end; (Bool) -> Bool - end, Config), + end, undefined, Config), ?assert_type(Config1, t()). -spec sweeper(list(), t()) -> t(). -sweeper(AppEnv, ConfigMap=#{sweeper := DefaultSweeperConfig}) -> +sweeper(AppEnv, ConfigMap) -> + DefaultSweeperConfig = maps:get(sweeper, ConfigMap, #{}), AppEnvSweeper = proplists:get_value(sweeper, AppEnv, #{}), %% convert sweeper config to a list to utilize the merge_list_with_environment function @@ -193,7 +249,7 @@ sweeper(AppEnv, ConfigMap=#{sweeper := DefaultSweeperConfig}) -> processors(AppEnv, ConfigMap) -> SpanProcessors = case transform(span_processor, proplists:get_value(span_processor, AppEnv)) of undefined -> - Processors = proplists:get_value(processors, AppEnv, maps:get(processors, ConfigMap)), + Processors = proplists:get_value(processors, AppEnv, maps:get(processors, ConfigMap, [])), transform(span_processors, Processors); SpanProcessor -> case proplists:get_value(processors, AppEnv) of @@ -212,7 +268,7 @@ processors(AppEnv, ConfigMap) -> {Name, Opts1} end, SpanProcessors), - ConfigMap#{processors := ProcessorsConfig}. + ConfigMap#{processors => ProcessorsConfig}. %% use the top level app env and os env configuration to set/override processor config values merge_processor_config(otel_batch_processor, Opts, ConfigMap, AppEnv) -> @@ -220,13 +276,11 @@ merge_processor_config(otel_batch_processor, Opts, ConfigMap, AppEnv) -> {bsp_exporting_timeout_ms, exporting_timeout_ms}, {bsp_max_queue_size, max_queue_size}, {traces_exporter, exporter}], - maps:merge(?BATCH_PROCESSOR_DEFAULTS, - merge_processor_config_(BatchEnvMapping, Opts, ConfigMap, AppEnv)); + merge_processor_config_(BatchEnvMapping, Opts, ConfigMap, AppEnv); merge_processor_config(otel_simple_processor, Opts, ConfigMap, AppEnv) -> SimpleEnvMapping = [{ssp_exporting_timeout_ms, exporting_timeout_ms}, {traces_exporter, exporter}], - maps:merge(?SIMPLE_PROCESSOR_DEFAULTS, - merge_processor_config_(SimpleEnvMapping, Opts, ConfigMap, AppEnv)); + merge_processor_config_(SimpleEnvMapping, Opts, ConfigMap, AppEnv); merge_processor_config(_, Opts, _, _) -> Opts. @@ -308,7 +362,7 @@ merge_list_with_environment(ConfigMappings, AppEnv, ConfigMap) -> update_config_map(OSVar, Key, Transform, Value, ConfigMap) -> try transform(Transform, Value) of TransformedValue -> - ConfigMap#{Key := TransformedValue} + ConfigMap#{Key => TransformedValue} catch Kind:Reason:StackTrace -> ?LOG_INFO(#{source => transform, @@ -533,9 +587,9 @@ transform(span_processors, Unknown) -> ?LOG_WARNING("processors value must be a list, but ~ts given. No span processors will be used", [Unknown]), []; transform(span_processor, batch) -> - {otel_batch_processor, ?BATCH_PROCESSOR_DEFAULTS}; + {otel_batch_processor, #{}}; transform(span_processor, simple) -> - {otel_simple_processor, ?SIMPLE_PROCESSOR_DEFAULTS}; + {otel_simple_processor, #{}}; transform(span_processor, SpanProcessor) -> SpanProcessor; transform(readers, Readers) -> diff --git a/apps/opentelemetry/src/otel_resource_detector.erl b/apps/opentelemetry/src/otel_resource_detector.erl index 24681e15..25f30138 100644 --- a/apps/opentelemetry/src/otel_resource_detector.erl +++ b/apps/opentelemetry/src/otel_resource_detector.erl @@ -82,10 +82,14 @@ get_resource(Timeout) -> end. %% @private -init([#{resource_detectors := Detectors, - resource_detector_timeout := DetectorTimeout}]) -> +init([Config]) -> process_flag(trap_exit, true), + Detectors = maps:get(resource_detectors, Config, [otel_resource_env_var, otel_resource_app_env]), + DetectorTimeout = maps:get(resource_detector_timeout, Config, 5000), + + + {ok, collecting, #data{resource=otel_resource:create([]), detectors=Detectors, detector_timeout=DetectorTimeout}, diff --git a/apps/opentelemetry/src/otel_span_limits.erl b/apps/opentelemetry/src/otel_span_limits.erl index 1fcd895b..3c70fbf1 100644 --- a/apps/opentelemetry/src/otel_span_limits.erl +++ b/apps/opentelemetry/src/otel_span_limits.erl @@ -36,12 +36,13 @@ get() -> persistent_term:get(?SPAN_LIMITS_KEY). -spec set(otel_configuration:t()) -> ok. -set(#{attribute_count_limit := AttributeCountLimit, - attribute_value_length_limit := AttributeValueLengthLimit, - event_count_limit := EventCountLimit, - link_count_limit := LinkCountLimit, - attribute_per_event_limit := AttributePerEventLimit, - attribute_per_link_limit := AttributePerLinkLimit}) -> +set(Config) -> + AttributeCountLimit = maps:get(attribute_count_limit, Config, 128), + AttributeValueLengthLimit = maps:get(attribute_value_length_limit, Config, infinity), + EventCountLimit = maps:get(event_count_limit, Config, 128), + LinkCountLimit = maps:get(link_count_limit, Config, 128), + AttributePerEventLimit = maps:get(attribute_per_event_limit, Config, 128), + AttributePerLinkLimit = maps:get(attribute_per_link_limit, Config, 128), SpanLimits = #span_limits{attribute_count_limit=AttributeCountLimit, attribute_value_length_limit=AttributeValueLengthLimit, event_count_limit=EventCountLimit, diff --git a/apps/opentelemetry/src/otel_span_sup.erl b/apps/opentelemetry/src/otel_span_sup.erl index 00999c9a..ad3b12d7 100644 --- a/apps/opentelemetry/src/otel_span_sup.erl +++ b/apps/opentelemetry/src/otel_span_sup.erl @@ -36,7 +36,7 @@ init([Config]) -> intensity => 1, period => 5}, - SweeperConfig = maps:get(sweeper, Config), + SweeperConfig = maps:get(sweeper, Config, #{}), Sweeper = #{id => otel_span_sweeper, start => {otel_span_sweeper, start_link, [SweeperConfig]}, restart => permanent, diff --git a/apps/opentelemetry/src/otel_span_sweeper.erl b/apps/opentelemetry/src/otel_span_sweeper.erl index 679438a1..09b8058c 100644 --- a/apps/opentelemetry/src/otel_span_sweeper.erl +++ b/apps/opentelemetry/src/otel_span_sweeper.erl @@ -50,10 +50,11 @@ storage_size() -> start_link(Config) -> gen_statem:start_link({local, ?MODULE}, ?MODULE, [Config], []). -init([#{interval := Interval, - strategy := Strategy, - span_ttl := TTL, - storage_size := StorageSize}]) -> +init([Config]) -> + Interval = maps:get(interval, Config, timer:minutes(10)), + Strategy = maps:get(strategy, Config, drop), + TTL = maps:get(span_ttl, Config, timer:minutes(30)), + StorageSize = maps:get(storage_size, Config, infinity), {ok, ready, #data{interval=Interval, strategy=Strategy, ttl=maybe_convert_time_unit(TTL), diff --git a/apps/opentelemetry/src/otel_tracer_server.erl b/apps/opentelemetry/src/otel_tracer_server.erl index 1d082daf..bf5f30f3 100644 --- a/apps/opentelemetry/src/otel_tracer_server.erl +++ b/apps/opentelemetry/src/otel_tracer_server.erl @@ -58,10 +58,12 @@ start_link(Name, RegName, SpanProcessorSupRegName, Resource, Config) -> gen_server:start_link({local, RegName}, ?MODULE, [Name, SpanProcessorSupRegName, Resource, Config], []). -init([Name, SpanProcessorSup, Resource, #{id_generator := IdGeneratorModule, - sampler := SamplerSpec, - processors := Processors, - deny_list := DenyList}]) -> +init([Name, SpanProcessorSup, Resource, Config]) -> + IdGeneratorModule = maps:get(id_generator, Config, otel_id_generator), + SamplerSpec = maps:get(sampler, Config, {parent_based, #{root => always_on}}), + Processors = maps:get(processors, Config, [{otel_batch_processor, #{}}]), + DenyList = maps:get(deny_list, Config, []), + Sampler = otel_sampler:new(SamplerSpec), Processors1 = init_processors(SpanProcessorSup, Processors), diff --git a/apps/opentelemetry/test/otel_configuration_SUITE.erl b/apps/opentelemetry/test/otel_configuration_SUITE.erl index e021c45b..67403493 100644 --- a/apps/opentelemetry/test/otel_configuration_SUITE.erl +++ b/apps/opentelemetry/test/otel_configuration_SUITE.erl @@ -198,9 +198,7 @@ end_per_testcase(_, Config) -> ok. empty_os_environment(_Config) -> - ?assertMatch(#{log_level := info, - text_map_propagators := [trace_context, baggage], - sampler := {parent_based, #{root := always_on}}}, + ?assertMatch(#{create_application_tracers := undefined}, otel_configuration:merge_with_os([])), ?assertMatch(#{log_level := error}, @@ -351,8 +349,7 @@ bad_span_limits(Config) -> compare_span_limits(Config). bad_app_config(_Config) -> - ?assertMatch(#{attribute_value_length_limit := infinity}, - otel_configuration:merge_with_os([{attribute_value_length_limit, "aaa"}])), + ?assertNot(maps:is_key(attribute_value_length_limit, otel_configuration:merge_with_os([{attribute_value_length_limit, "aaa"}]))), ok. @@ -376,8 +373,7 @@ span_processors(_Config) -> ?assertMatch(#{processors := [{otel_simple_processor, #{}}]}, otel_configuration:merge_with_os([{span_processor, simple}])), - ?assertMatch(#{processors := [{otel_batch_processor, #{exporter := {opentelemetry_exporter,#{}}, - exporting_timeout_ms := 2, + ?assertMatch(#{processors := [{otel_batch_processor, #{exporting_timeout_ms := 2, max_queue_size := 1, scheduled_delay_ms := 15000}}]}, otel_configuration:merge_with_os([{span_processor, batch}, @@ -389,10 +385,7 @@ span_processors(_Config) -> otel_configuration:merge_with_os([{span_processor, batch}, {traces_exporter, none}])), - ?assertMatch(#{processors := [{otel_batch_processor, #{exporter := {opentelemetry_exporter,#{}}, - exporting_timeout_ms := 30000, - max_queue_size := 2048, - scheduled_delay_ms := 5000}}]}, + ?assertMatch(#{processors := [{otel_batch_processor, #{}}]}, otel_configuration:merge_with_os([{span_processor, batch}])), ?assertMatch(#{processors := [{otel_batch_processor, #{exporter := {opentelemetry_exporter, @@ -407,8 +400,7 @@ span_processors(_Config) -> {bsp_exporting_timeout_ms, 2}, {bsp_max_queue_size, 1}])), - ?assertMatch(#{processors := [{otel_simple_processor, #{exporter := {opentelemetry_exporter,#{}}, - exporting_timeout_ms := 2}}]}, + ?assertMatch(#{processors := [{otel_simple_processor, #{exporting_timeout_ms := 2}}]}, otel_configuration:merge_with_os([{span_processor, simple}, {ssp_exporting_timeout_ms, 2}])), diff --git a/apps/opentelemetry_api/include/opentelemetry.hrl b/apps/opentelemetry_api/include/opentelemetry.hrl index cd59d3aa..50dca439 100644 --- a/apps/opentelemetry_api/include/opentelemetry.hrl +++ b/apps/opentelemetry_api/include/opentelemetry.hrl @@ -18,6 +18,8 @@ -define(REG_NAME(Name), list_to_atom(lists:concat([?MODULE, "_", Name]))). -define(GLOBAL_TRACER_PROVIDER_NAME, global). -define(GLOBAL_TRACER_PROVIDER_REG_NAME, otel_tracer_provider_global). +-define(GLOBAL_CONFIG_PROVIDER_NAME, global). +-define(GLOBAL_CONFIG_PROVIDER_REG_NAME, otel_config_provider_global). %% These records are based on protos found in the opentelemetry-proto repo: %% src/opentelemetry/proto/trace/v1/trace.proto diff --git a/config/sdk.config b/config/sdk.config index e9febbb6..94092949 100644 --- a/config/sdk.config +++ b/config/sdk.config @@ -2,18 +2,14 @@ {file_format, "1.0-rc.1"}, {disabled, false}, {log_level, info}, - {resource, [ - {attributes, [ + {resource, #{attributes, [ #{name => <<"service.name">>, value => <<"unknown_service">>} - ]} - ]}, - {attribute_limits, [ - {attribute_value_length_limit, undefined}, - {attribute_count_limit, 128} - ]}, - {propagator, [ - {composite, [tracecontext, baggage]} - ]}, + ]}}, + {attribute_limits, #{attribute_value_length_limit => undefined, + attribute_count_limit => 128}}, + + {propagator, #{composite => [tracecontext, baggage], + composite_list => ""}, {tracer_provider, [ {processors, [ From 849503e7fb2c1b7c5fdc8a60e723a401d4d86b3c Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Thu, 3 Jul 2025 06:43:14 -0400 Subject: [PATCH 03/15] use new config property attribute_limits for configuring span limits --- apps/opentelemetry/src/otel_configuration.erl | 19 ++++++++++++++++++- apps/opentelemetry/src/otel_span_limits.erl | 6 ++++-- .../test/otel_configuration_SUITE.erl | 4 ++-- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/apps/opentelemetry/src/otel_configuration.erl b/apps/opentelemetry/src/otel_configuration.erl index f34a74f4..0e65f58b 100644 --- a/apps/opentelemetry/src/otel_configuration.erl +++ b/apps/opentelemetry/src/otel_configuration.erl @@ -163,7 +163,7 @@ merge_with_os(AppEnv) -> -spec convert_to_new(#{}) -> t(). convert_to_new(OldConfig) -> - convert_disabled(OldConfig). + convert_attribute_limits(convert_disabled(OldConfig)). convert_disabled(OldConfig) -> case maps:take(sdk_disabled, OldConfig) of @@ -173,6 +173,23 @@ convert_disabled(OldConfig) -> OldConfig end. +convert_attribute_limits(OldConfig) -> + OldConfig2 = case maps:take(attribute_count_limit, OldConfig) of + {AttributeCountLimit, OldConfig1} -> + OldConfig1#{attribute_limits => #{attribute_count_limit => AttributeCountLimit}}; + error -> + OldConfig + end, + + + case maps:take(attribute_value_length_limit, OldConfig2) of + {AttributeValueLengthLimit, OldConfig3} -> + AttributeLimitsMap = maps:get(attribute_limits, OldConfig3, #{}), + OldConfig3#{attribute_limits => AttributeLimitsMap#{attribute_value_length_limit => AttributeValueLengthLimit}}; + error -> + OldConfig2 + end. + %% ConfigMap = new(), %% ConfigMap1 = lists:foldl(fun(F, Acc) -> diff --git a/apps/opentelemetry/src/otel_span_limits.erl b/apps/opentelemetry/src/otel_span_limits.erl index 3c70fbf1..7b5cf718 100644 --- a/apps/opentelemetry/src/otel_span_limits.erl +++ b/apps/opentelemetry/src/otel_span_limits.erl @@ -37,8 +37,10 @@ get() -> -spec set(otel_configuration:t()) -> ok. set(Config) -> - AttributeCountLimit = maps:get(attribute_count_limit, Config, 128), - AttributeValueLengthLimit = maps:get(attribute_value_length_limit, Config, infinity), + AttributeLimits = maps:get(attribute_limits, Config, #{}), + AttributeCountLimit = maps:get(attribute_count_limit, AttributeLimits, 128), + AttributeValueLengthLimit = maps:get(attribute_value_length_limit, AttributeLimits, infinity), + EventCountLimit = maps:get(event_count_limit, Config, 128), LinkCountLimit = maps:get(link_count_limit, Config, 128), AttributePerEventLimit = maps:get(attribute_per_event_limit, Config, 128), diff --git a/apps/opentelemetry/test/otel_configuration_SUITE.erl b/apps/opentelemetry/test/otel_configuration_SUITE.erl index 67403493..704b208c 100644 --- a/apps/opentelemetry/test/otel_configuration_SUITE.erl +++ b/apps/opentelemetry/test/otel_configuration_SUITE.erl @@ -149,8 +149,8 @@ init_per_testcase(span_limits, Config) -> setup_env(Vars), - ExpectedOpts = #{attribute_count_limit => 111, - attribute_value_length_limit => 9, + ExpectedOpts = #{attribute_limits => #{attribute_count_limit => 111, + attribute_value_length_limit => 9}, event_count_limit => 200, link_count_limit => 1101, attribute_per_event_limit => 400, From 13eccc71d00013a0cb5c8dca25bcea60302682ea Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Sat, 5 Jul 2025 06:43:20 -0400 Subject: [PATCH 04/15] support declarative file config form of resource attributes --- CHANGELOG.md | 11 +- apps/opentelemetry/src/otel_configuration.erl | 38 +++- .../src/otel_resource_app_env.erl | 40 ++-- .../src/otel_resource_detector.erl | 14 +- .../test/otel_resource_SUITE.erl | 184 +++++++++++------- config/sdk.config | 6 +- 6 files changed, 194 insertions(+), 99 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f4650775..fd12d504 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,11 +9,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### SDK -### +#### Added -- []() +- -### Changes +#### Changes + +- *Deprecated*: Resource attribute configuration: No longer use maps to + structure resource attribute keys with dots. Meaning use a key of + `service.name` instead of `#{service => #{name => ...}}` to represent + `service.name` attribute in the resource. ## Experimental API 0.5.2 - 2024-11-22 diff --git a/apps/opentelemetry/src/otel_configuration.erl b/apps/opentelemetry/src/otel_configuration.erl index 0e65f58b..4c502cac 100644 --- a/apps/opentelemetry/src/otel_configuration.erl +++ b/apps/opentelemetry/src/otel_configuration.erl @@ -163,7 +163,43 @@ merge_with_os(AppEnv) -> -spec convert_to_new(#{}) -> t(). convert_to_new(OldConfig) -> - convert_attribute_limits(convert_disabled(OldConfig)). + convert_resource( + convert_attribute_limits( + convert_disabled(OldConfig))). + +%% this one is different, it wasn't read from the old config before, only from application env by the detector +%% that detector is now a no-op and its parsing is done here. The old supported option for +%% adding static resource attributes is combined with the new. Anything under the key +%% `attributes' is expected to be of the form `#{name := binary(), value := binary()}' while +%% every other key/value pair is assumed to be an attribute +convert_resource(OldConfig) -> + case application:get_env(opentelemetry, resource) of + {ok, ResourceAttributes} when is_list(ResourceAttributes) orelse is_map(ResourceAttributes) -> + ResourceAttributes0 = case is_list(ResourceAttributes) of + true -> + maps:from_list(ResourceAttributes); + false -> + ResourceAttributes + end, + {NewConfigAttributes1, ResourceAttributes2} + = try maps:take(attributes, ResourceAttributes0) of + {NewConfigAttributes, ResourceAttributes1} -> + {NewConfigAttributes, ResourceAttributes1}; + error -> + {[], ResourceAttributes0} + catch + %% old form of resource attributes may not be a map but instead a list + %% we don't bother looking for an attributes key in that list + _:_ -> + {[], ResourceAttributes0} + end, + + ParsedAttributes = otel_resource_app_env:parse(ResourceAttributes2), + + OldConfig#{resource => #{attributes => ParsedAttributes ++ NewConfigAttributes1}}; + _ -> + OldConfig + end. convert_disabled(OldConfig) -> case maps:take(sdk_disabled, OldConfig) of diff --git a/apps/opentelemetry/src/otel_resource_app_env.erl b/apps/opentelemetry/src/otel_resource_app_env.erl index a30daea6..fbf5c3a3 100644 --- a/apps/opentelemetry/src/otel_resource_app_env.erl +++ b/apps/opentelemetry/src/otel_resource_app_env.erl @@ -12,7 +12,7 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %% -%% @doc Resource detector ({@link otel_resource_detector}) which adds attributes +%% @doc DEPRECATED: Resource detector ({@link otel_resource_detector}) which adds attributes %% to the `Resource' based on the value of `resource' %% in the `opentelemetry' application's environment. %% @@ -38,10 +38,10 @@ -export([get_resource/1, parse/1]). +%% now a no-op. `resource` is read from application environment by configuration instead %% @private get_resource(_Config) -> - Attributes = parse(application:get_env(opentelemetry, resource, #{})), - otel_resource:create(Attributes). + otel_resource:create([]). %% @@ -49,7 +49,11 @@ get_resource(_Config) -> parse(Attributes) when is_map(Attributes) -> parse(maps:to_list(Attributes)); parse(Attributes) when is_list(Attributes) -> - lists:flatmap(fun({Key, Values}) when is_list(Key) ; is_binary(Key) ; is_atom(Key) -> + lists:flatmap(fun(#{name := Key, + value := Value}) -> + [#{name => unicode:characters_to_binary(Key), + value => Value}]; + ({Key, Values}) when is_list(Key) ; is_binary(Key) ; is_atom(Key) -> parse_values(to_string(Key), Values); (_) -> %% ignore anything else @@ -64,20 +68,26 @@ parse_values(Key, Values) when is_map(Values) -> parse_values(Key, Values) when is_list(Values) -> case io_lib:printable_unicode_list(Values) of true -> - [{unicode:characters_to_binary(Key), unicode:characters_to_binary(Values)}]; + [#{name => unicode:characters_to_binary(Key), + value => unicode:characters_to_binary(Values)}]; false -> - lists:flatmap(fun({SubKey, Value=[{_,_}|_]}) -> - %% list of tuples means we have more subkeys - parse_values([Key, ".", to_string(SubKey)], Value); - ({SubKey, Value}) when is_map(Value) -> - %% map value means we have more subkeys - parse_values([Key, ".", to_string(SubKey)], Value); - ({SubKey, Value})-> - [{otel_utils:assert_to_binary([Key, ".", to_string(SubKey)]), Value}] - end, Values) + lists:flatmap( + %% the map of name/value is from the otel declarative configuration file spec + %% and replaces the old methods below it + fun({SubKey, Value=[{_,_}|_]}) -> + %% list of tuples means we have more subkeys + parse_values([Key, ".", to_string(SubKey)], Value); + ({SubKey, Value}) when is_map(Value) -> + %% map value means we have more subkeys + parse_values([Key, ".", to_string(SubKey)], Value); + ({SubKey, Value})-> + [#{name => otel_utils:assert_to_binary([Key, ".", to_string(SubKey)]), + value => Value}] + end, Values) end; parse_values(Key, Value) -> - [{unicode:characters_to_binary(Key), Value}]. + [#{name => unicode:characters_to_binary(Key), + value => Value}]. -spec to_string(atom() | binary() | list()) -> binary(). to_string(K) when is_atom(K) -> diff --git a/apps/opentelemetry/src/otel_resource_detector.erl b/apps/opentelemetry/src/otel_resource_detector.erl index 25f30138..32cd7925 100644 --- a/apps/opentelemetry/src/otel_resource_detector.erl +++ b/apps/opentelemetry/src/otel_resource_detector.erl @@ -85,12 +85,14 @@ get_resource(Timeout) -> init([Config]) -> process_flag(trap_exit, true), - Detectors = maps:get(resource_detectors, Config, [otel_resource_env_var, otel_resource_app_env]), + Detectors = maps:get(resource_detectors, Config, [otel_resource_env_var]), DetectorTimeout = maps:get(resource_detector_timeout, Config, 5000), + ConfiguredAttributes = maps:get(attributes, + maps:get(resource, Config, #{}), []), + AttributesMap = configured_attributes_to_map(ConfiguredAttributes), - - {ok, collecting, #data{resource=otel_resource:create([]), + {ok, collecting, #data{resource=otel_resource:create(AttributesMap), detectors=Detectors, detector_timeout=DetectorTimeout}, [{next_event, internal, spawn_detectors}]}. @@ -142,6 +144,12 @@ handle_event(_, _, ready, _) -> %% +configured_attributes_to_map(ConfiguredAttributes) -> + lists:foldl(fun(#{name := Name, + value := Value}, Acc) -> + Acc#{Name => Value} + end, #{}, ConfiguredAttributes). + %% go to the `ready' state if the list of detectors to collect for is empty next_state([]) -> ready; diff --git a/apps/opentelemetry/test/otel_resource_SUITE.erl b/apps/opentelemetry/test/otel_resource_SUITE.erl index 9bfd4243..b06edefd 100644 --- a/apps/opentelemetry/test/otel_resource_SUITE.erl +++ b/apps/opentelemetry/test/otel_resource_SUITE.erl @@ -17,7 +17,7 @@ all() -> combining_conflicting_schemas, crash_detector, timeout_detector, release_service_name, unknown_service_name, release_service_name_no_version, service_instance_id_env, service_instance_id_env_attributes, {group, net_kernel_node_name}, service_instance_id_node_id2, - validate_keys, do_not_create_unneeded_atoms]. + validate_keys, do_not_create_unneeded_atoms, deprecated_attributes_env, deprecated_attributes_env_in_list]. groups() -> [{net_kernel_node_name, [], [service_instance_id_node_id1, @@ -60,6 +60,10 @@ end_per_testcase(_, _) -> startup(_Config) -> try + application:load(opentelemetry), + application:set_env(opentelemetry, resource, #{attributes => [#{name => <<"attr-1">>, + value => <<"value-1">>}]}), + os:putenv("OTEL_RESOURCE_ATTRIBUTES", "service.name=cttest,service.version=1.1.1"), {ok, _} = application:ensure_all_started(opentelemetry), @@ -67,7 +71,65 @@ startup(_Config) -> _ = application:stop(opentelemetry), ?assertMatch(#{'service.name' := <<"cttest">>, - 'service.version' := <<"1.1.1">>}, otel_attributes:map(otel_resource:attributes(Resource))), + 'service.version' := <<"1.1.1">>, + 'attr-1' := <<"value-1">>}, otel_attributes:map(otel_resource:attributes(Resource))), + ok + after + os:unsetenv("OTEL_RESOURCE_ATTRIBUTES"), + application:stop(opentelemetry), + application:unload(opentelemetry) + end. + +%% check the merging of the old and the new form of static resource attributes in the +%% app env +deprecated_attributes_env(_Config) -> + try + application:load(opentelemetry), + %% a combination of the old and the new + application:set_env(opentelemetry, resource, #{attributes => [#{name => <<"attr-1">>, + value => <<"value-1">>}], + <<"c">> => <<"d">>, + "sk" => "sv" }), + + os:putenv("OTEL_RESOURCE_ATTRIBUTES", "service.name=cttest,service.version=1.1.1"), + + {ok, _} = application:ensure_all_started(opentelemetry), + Resource = otel_tracer_provider:resource(), + _ = application:stop(opentelemetry), + + ?assertMatch(#{'service.name' := <<"cttest">>, + 'service.version' := <<"1.1.1">>, + 'attr-1' := <<"value-1">>, + c := <<"d">>, + sk := <<"sv">>}, otel_attributes:map(otel_resource:attributes(Resource))), + ok + after + os:unsetenv("OTEL_RESOURCE_ATTRIBUTES"), + application:stop(opentelemetry), + application:unload(opentelemetry) + end. + +%% check the merging of the old and the new when in a list +deprecated_attributes_env_in_list(_Config) -> + try + application:load(opentelemetry), + %% a combination of the old and the new + application:set_env(opentelemetry, resource, [{attributes, [#{name => <<"attr-1">>, + value => <<"value-1">>}]}, + {<<"c">>, <<"d">>}, + {"sk", "sv" }]), + + os:putenv("OTEL_RESOURCE_ATTRIBUTES", "service.name=cttest,service.version=1.1.1"), + + {ok, _} = application:ensure_all_started(opentelemetry), + Resource = otel_tracer_provider:resource(), + _ = application:stop(opentelemetry), + + ?assertMatch(#{'service.name' := <<"cttest">>, + 'service.version' := <<"1.1.1">>, + 'attr-1' := <<"value-1">>, + c := <<"d">>, + sk := <<"sv">>}, otel_attributes:map(otel_resource:attributes(Resource))), ok after os:unsetenv("OTEL_RESOURCE_ATTRIBUTES"), @@ -98,21 +160,16 @@ startup_env_service_name(_Config) -> crash_detector(_Config) -> try application:load(opentelemetry), - application:set_env(opentelemetry, resource, #{<<"c">> => <<"d">>, - "sk" => "sv"}), os:putenv("OTEL_RESOURCE_ATTRIBUTES", "service.name=cttest,service.version=2.1.1"), - otel_resource_detector:start_link(#{resource_detectors => [otel_resource_env_var, - {otel_resource_detector_test, error}, - otel_resource_app_env], + otel_resource_detector:start_link(#{resource_detectors => [{otel_resource_detector_test, error}, + otel_resource_env_var], resource_detector_timeout => 100}), Resource = otel_resource_detector:get_resource(), ?assertMatch(#{'service.name' := <<"cttest">>, - 'service.version' := <<"2.1.1">>, - c := <<"d">>, - sk := <<"sv">>}, otel_attributes:map(otel_resource:attributes(Resource))), + 'service.version' := <<"2.1.1">>}, otel_attributes:map(otel_resource:attributes(Resource))), ok after @@ -127,15 +184,13 @@ timeout_detector(_Config) -> os:putenv("OTEL_RESOURCE_ATTRIBUTES", "service.name=cttest,service.version=3.1.1"), otel_resource_detector:start_link(#{resource_detectors => [otel_resource_env_var, - {otel_resource_detector_test, sleep}, - otel_resource_app_env], + {otel_resource_detector_test, sleep}], resource_detector_timeout => 100}), Resource = otel_resource_detector:get_resource(), ?assertMatch(#{'service.name' := <<"cttest">>, - 'service.version' := <<"3.1.1">>, - e := <<"f">>}, otel_attributes:map(otel_resource:attributes(Resource))), + 'service.version' := <<"3.1.1">>}, otel_attributes:map(otel_resource:attributes(Resource))), ?assertEqual(otel_resource:create([]), otel_resource_detector:get_resource(0)), @@ -153,7 +208,7 @@ os_env_resource(_Config) -> app_env_resource(_Config) -> Attributes = #{a => [{b,[{c,d}]}], service => #{name => <<"hello">>}}, - Expected = [{<<"a.b.c">>, d}, {<<"service.name">>, <<"hello">>}], + Expected = [#{name => <<"a.b.c">>, value => d}, #{name => <<"service.name">>, value => <<"hello">>}], %% sort because this is created from a map and need to make sure %% the order is always the same when we do the assertion @@ -161,9 +216,9 @@ app_env_resource(_Config) -> ok. combining(_Config) -> - Resource1 = otel_resource:create(otel_resource_app_env:parse([{service, [{name, <<"other-name">>}, - {version, "1.1.1"}]}]), - <<"https://opentelemetry.io/schemas/1.8.0">>), + Resource1 = create_from_config(otel_resource_app_env:parse([{service, [{name, <<"other-name">>}, + {version, "1.1.1"}]}]), + <<"https://opentelemetry.io/schemas/1.8.0">>), Resource2 = otel_resource:create(otel_resource_env_var:parse("service.name=cttest,service.version=1.1.1"), <<"https://opentelemetry.io/schemas/1.8.0">>), @@ -176,8 +231,9 @@ combining(_Config) -> ok. combining_conflicting_schemas(_Config) -> - Resource1 = otel_resource:create(otel_resource_app_env:parse([{service, [{name, <<"other-name">>}, - {version, "1.1.1"}]}]), <<"https://opentelemetry.io/schemas/1.8.0">>), + Resource1 = create_from_config([{service, [{name, <<"other-name">>}, + {version, "1.1.1"}]}], + <<"https://opentelemetry.io/schemas/1.8.0">>), Resource2 = otel_resource:create(otel_resource_env_var:parse("service.name=cttest,service.version=1.1.1"), <<"https://opentelemetry.io/schemas/1.7.0">>), @@ -193,19 +249,13 @@ unknown_service_name(_Config) -> try os:unsetenv("OTEL_RESOURCE_ATTRIBUTES"), - application:unload(opentelemetry), - application:load(opentelemetry), - application:set_env(opentelemetry, resource, #{<<"e">> => <<"f">>}), - - otel_resource_detector:start_link(#{resource_detectors => [otel_resource_env_var, - otel_resource_app_env], + otel_resource_detector:start_link(#{resource_detectors => [otel_resource_env_var], resource_detector_timeout => 100}), Resource = otel_resource_detector:get_resource(), ?assertMatch(#{'service.name' := <<"unknown_service:erl">>, 'process.runtime.name' := <<"BEAM">>, - 'process.executable.name' := <<"erl">>, - e := <<"f">>}, otel_attributes:map(otel_resource:attributes(Resource))), + 'process.executable.name' := <<"erl">>}, otel_attributes:map(otel_resource:attributes(Resource))), ok after @@ -216,18 +266,14 @@ release_service_name(_Config) -> try os:putenv("RELEASE_NAME", "rel-cttest"), os:putenv("RELEASE_VSN", "0.1.0"), - application:unload(opentelemetry), application:load(opentelemetry), - application:set_env(opentelemetry, resource, #{<<"e">> => <<"f">>}), - otel_resource_detector:start_link(#{resource_detectors => [otel_resource_env_var, - otel_resource_app_env], + otel_resource_detector:start_link(#{resource_detectors => [otel_resource_env_var], resource_detector_timeout => 100}), Resource = otel_resource_detector:get_resource(), ?assertMatch(#{'service.name' := <<"rel-cttest">>, - 'service.version' := <<"0.1.0">>, - e := <<"f">>}, otel_attributes:map(otel_resource:attributes(Resource))), + 'service.version' := <<"0.1.0">>}, otel_attributes:map(otel_resource:attributes(Resource))), ok after @@ -238,17 +284,13 @@ release_service_name(_Config) -> service_instance_id_env(_Config) -> try os:putenv("OTEL_SERVICE_INSTANCE", "test@instance"), - application:unload(opentelemetry), application:load(opentelemetry), - application:set_env(opentelemetry, resource, #{<<"e">> => <<"f">>}), - otel_resource_detector:start_link(#{resource_detectors => [otel_resource_env_var, - otel_resource_app_env], + otel_resource_detector:start_link(#{resource_detectors => [otel_resource_env_var], resource_detector_timeout => 100}), Resource = otel_resource_detector:get_resource(), - ?assertMatch(#{'service.instance.id' := <<"test@instance">>, - e := <<"f">>}, otel_attributes:map(otel_resource:attributes(Resource))), + ?assertMatch(#{'service.instance.id' := <<"test@instance">>}, otel_attributes:map(otel_resource:attributes(Resource))), ok after @@ -258,17 +300,14 @@ service_instance_id_env(_Config) -> service_instance_id_env_attributes(_Config) -> try os:putenv("OTEL_RESOURCE_ATTRIBUTES", "service.instance.id=test@instance"), - application:unload(opentelemetry), application:load(opentelemetry), - application:set_env(opentelemetry, resource, #{<<"e">> => <<"f">>}), - otel_resource_detector:start_link(#{resource_detectors => [otel_resource_env_var, - otel_resource_app_env], - resource_detector_timeout => 100}), + %% resource_app_env is now a noop so this will not result in e => f in the attributes + otel_resource_detector:start_link(#{resource_detectors => [otel_resource_env_var], + resource_detector_timekout => 100}), Resource = otel_resource_detector:get_resource(), - ?assertMatch(#{'service.instance.id' := <<"test@instance">>, - e := <<"f">>}, otel_attributes:map(otel_resource:attributes(Resource))), + ?assertMatch(#{'service.instance.id' := <<"test@instance">>}, otel_attributes:map(otel_resource:attributes(Resource))), ok after @@ -277,8 +316,7 @@ service_instance_id_env_attributes(_Config) -> service_instance_id_node_name(_Config) -> Resource = otel_resource_detector:get_resource(), - ?assertMatch(#{'service.instance.id' := <<"test@instance">>, - e := <<"f">>}, otel_attributes:map(otel_resource:attributes(Resource))), + ?assertMatch(#{'service.instance.id' := <<"test@instance">>}, otel_attributes:map(otel_resource:attributes(Resource))), ok. @@ -291,12 +329,8 @@ service_instance_id_node_id1(_Config) -> ok. service_instance_id_node_id2(_Config) -> - application:unload(opentelemetry), application:load(opentelemetry), - application:set_env(opentelemetry, resource, #{<<"e">> => <<"f">>}), - - otel_resource_detector:start_link(#{resource_detectors => [otel_resource_env_var, - otel_resource_app_env], + otel_resource_detector:start_link(#{resource_detectors => [otel_resource_env_var], resource_detector_timeout => 100}), Resource = otel_resource_detector:get_resource(), @@ -308,17 +342,13 @@ service_instance_id_node_id2(_Config) -> release_service_name_no_version(_Config) -> try os:putenv("RELEASE_NAME", "rel-cttest"), - application:unload(opentelemetry), application:load(opentelemetry), - application:set_env(opentelemetry, resource, #{<<"e">> => <<"f">>}), - otel_resource_detector:start_link(#{resource_detectors => [otel_resource_env_var, - otel_resource_app_env], + otel_resource_detector:start_link(#{resource_detectors => [otel_resource_env_var], resource_detector_timeout => 100}), Resource = otel_resource_detector:get_resource(), - ?assertMatch(#{'service.name' := <<"rel-cttest">>, - e := <<"f">>}, otel_attributes:map(otel_resource:attributes(Resource))), + ?assertMatch(#{'service.name' := <<"rel-cttest">>}, otel_attributes:map(otel_resource:attributes(Resource))), ?assertNot(maps:is_key('service.version', otel_attributes:map(otel_resource:attributes(Resource)))), ok @@ -329,21 +359,19 @@ release_service_name_no_version(_Config) -> validate_keys(_Config) -> application:unload(opentelemetry), application:load(opentelemetry), - application:set_env(opentelemetry, resource, #{ - <<"e">> => <<"f">>, - service => #{ - <<"name">> => <<"name">>, - alias => <<"alias">> - }, - g => <<"h">>, - '' => <<"i">>, - 'É™' => <<"j">> - }), - - otel_resource_detector:start_link(#{resource_detectors => [otel_resource_app_env], - resource_detector_timeout => 100}), + ConfigAttributes = #{ + <<"e">> => <<"f">>, + service => #{ + <<"name">> => <<"name">>, + alias => <<"alias">> + }, + g => <<"h">>, + '' => <<"i">>, + 'É™' => <<"j">> + }, + + Resource = create_from_config(ConfigAttributes, <<"">>), - Resource = otel_resource_detector:get_resource(), ?assert(otel_resource:is_key(e, Resource)), ?assert(otel_resource:is_key('service.name', Resource)), ?assert(otel_resource:is_key(<<"service.alias">>, Resource)), @@ -384,3 +412,11 @@ start_net_kernel_and_detector(NetKernelArgs) -> otel_resource_app_env], resource_detector_timeout => 100}), ok. + +create_from_config(ConfigAttributes, Schema) -> + Parsed = otel_resource_app_env:parse(ConfigAttributes), + + Attributes = [{Name, Value} || #{name := Name, + value := Value} <- Parsed], + + otel_resource:create(Attributes, Schema). diff --git a/config/sdk.config b/config/sdk.config index 94092949..81c0fe2c 100644 --- a/config/sdk.config +++ b/config/sdk.config @@ -2,9 +2,9 @@ {file_format, "1.0-rc.1"}, {disabled, false}, {log_level, info}, - {resource, #{attributes, [ - #{name => <<"service.name">>, value => <<"unknown_service">>} - ]}}, + {resource, #{attributes => [ + #{name => <<"service.name">>, value => <<"unknown_service">>} + ]}}, {attribute_limits, #{attribute_value_length_limit => undefined, attribute_count_limit => 128}}, From a9e9b6a3d31dc500bec1929ad8af18bbb15cdf43 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Sat, 5 Jul 2025 12:45:52 -0400 Subject: [PATCH 05/15] support new config file form of propagator configuring --- apps/opentelemetry/src/opentelemetry_app.erl | 19 ++++++++- apps/opentelemetry/src/otel_configuration.erl | 39 +++++++++++++------ apps/opentelemetry/src/otel_resource.erl | 2 +- .../test/opentelemetry_SUITE.erl | 18 ++++++++- .../test/otel_configuration_SUITE.erl | 6 +-- 5 files changed, 66 insertions(+), 18 deletions(-) diff --git a/apps/opentelemetry/src/opentelemetry_app.erl b/apps/opentelemetry/src/opentelemetry_app.erl index 7b6151c1..4a3de242 100644 --- a/apps/opentelemetry/src/opentelemetry_app.erl +++ b/apps/opentelemetry/src/opentelemetry_app.erl @@ -58,8 +58,13 @@ stop(_State) -> %% internal functions setup_text_map_propagators(Config) -> - List = maps:get(text_map_propagators, Config, []), - CompositePropagator = otel_propagator_text_map_composite:create(List), + Propagator = maps:get(propagator, Config, #{}), + PropagatorList = maps:get(composite_list, Propagator, []), + Propagators = maps:get(composite, Propagator, []), + + Combined = append_if_not_found(otel_configuration:transform(propagators, PropagatorList), Propagators), + + CompositePropagator = otel_propagator_text_map_composite:create(Combined), opentelemetry:set_text_map_propagator(CompositePropagator). create_loaded_application_tracers(#{create_application_tracers := true}) -> @@ -69,3 +74,13 @@ create_loaded_application_tracers(#{create_application_tracers := true}) -> ok; create_loaded_application_tracers(_) -> ok. + +append_if_not_found([], Existing) -> + Existing; +append_if_not_found([C | Rest], Existing) -> + case lists:member(C, Existing) of + true -> + append_if_not_found(Rest, Existing); + false -> + append_if_not_found(Rest, Existing ++ [C]) + end. diff --git a/apps/opentelemetry/src/otel_configuration.erl b/apps/opentelemetry/src/otel_configuration.erl index 4c502cac..4bf1b0c1 100644 --- a/apps/opentelemetry/src/otel_configuration.erl +++ b/apps/opentelemetry/src/otel_configuration.erl @@ -21,6 +21,7 @@ -export([merge_with_os/1, merge_list_with_environment/3, + transform/2, report_cb/1]). -type log_level() :: atom(). @@ -150,29 +151,45 @@ new() -> %% constructs a config based on the flat format and merges with the new %% nested configuration from the declarative configuration SIG -spec merge_with_os(list()) -> t(). -merge_with_os(AppEnv) -> +merge_with_os(Config) -> OldConfig = lists:foldl(fun(F, Acc) -> - F(AppEnv, Acc) + F(Config, Acc) end, #{}, [fun span_limits/2, fun general/2, fun sampler/2, fun processors/2, fun sweeper/2]), - convert_to_new(OldConfig). + convert_to_new(OldConfig, Config). --spec convert_to_new(#{}) -> t(). -convert_to_new(OldConfig) -> - convert_resource( - convert_attribute_limits( - convert_disabled(OldConfig))). +-spec convert_to_new(#{}, map()) -> t(). +convert_to_new(OldConfig, Config) -> + convert_propagator( + convert_resource( + convert_attribute_limits( + convert_disabled(OldConfig, Config), Config), Config), Config). + +%% convert the old `text_map_propagators' config into the new `propagator' config +%% based on the declarative file configuration of otel +convert_propagator(OldConfig, Config) -> + case lists:keyfind(propagator, 1, Config) of + false -> + case maps:take(text_map_propagators, OldConfig) of + {Value, OldConfig1} -> + OldConfig1#{propagator => #{composite => Value}}; + error -> + OldConfig + end; + {propagator, Value} -> + OldConfig#{propagator => Value} + end. %% this one is different, it wasn't read from the old config before, only from application env by the detector %% that detector is now a no-op and its parsing is done here. The old supported option for %% adding static resource attributes is combined with the new. Anything under the key %% `attributes' is expected to be of the form `#{name := binary(), value := binary()}' while %% every other key/value pair is assumed to be an attribute -convert_resource(OldConfig) -> +convert_resource(OldConfig, Config) -> case application:get_env(opentelemetry, resource) of {ok, ResourceAttributes} when is_list(ResourceAttributes) orelse is_map(ResourceAttributes) -> ResourceAttributes0 = case is_list(ResourceAttributes) of @@ -201,7 +218,7 @@ convert_resource(OldConfig) -> OldConfig end. -convert_disabled(OldConfig) -> +convert_disabled(OldConfig, Config) -> case maps:take(sdk_disabled, OldConfig) of {Value, OldConfig1} -> OldConfig1#{disabled => Value}; @@ -209,7 +226,7 @@ convert_disabled(OldConfig) -> OldConfig end. -convert_attribute_limits(OldConfig) -> +convert_attribute_limits(OldConfig, Config) -> OldConfig2 = case maps:take(attribute_count_limit, OldConfig) of {AttributeCountLimit, OldConfig1} -> OldConfig1#{attribute_limits => #{attribute_count_limit => AttributeCountLimit}}; diff --git a/apps/opentelemetry/src/otel_resource.erl b/apps/opentelemetry/src/otel_resource.erl index 5c79c594..110c7109 100644 --- a/apps/opentelemetry/src/otel_resource.erl +++ b/apps/opentelemetry/src/otel_resource.erl @@ -123,7 +123,7 @@ is_key(_, _) -> -spec merge(t(), t()) -> t(). merge(#resource{schema_url=NewSchemaUrl, attributes=NewAttributes}, CurrentResource=#resource{schema_url=CurrentSchemaUrl, - attributes=CurrentAttributes}) -> + attributes=CurrentAttributes}) -> SchameUrl = merge_schema_url(NewSchemaUrl, CurrentSchemaUrl), NewMap = otel_attributes:map(NewAttributes), CurrentResource#resource{schema_url=SchameUrl, diff --git a/apps/opentelemetry/test/opentelemetry_SUITE.erl b/apps/opentelemetry/test/opentelemetry_SUITE.erl index 5a7deb95..a9373818 100644 --- a/apps/opentelemetry/test/opentelemetry_SUITE.erl +++ b/apps/opentelemetry/test/opentelemetry_SUITE.erl @@ -39,7 +39,8 @@ all_cases() -> root_span_sampling_always_on, root_span_sampling_always_off, record_but_not_sample, record_exception_works, record_exception_with_message_works, propagator_configuration, propagator_configuration_with_os_env, force_flush, - dropped_attributes, too_many_attributes, truncated_binary_attributes]. + dropped_attributes, too_many_attributes, truncated_binary_attributes, + new_propagator_configuration]. groups() -> [{otel_simple_processor, [], all_cases()}, @@ -95,6 +96,12 @@ init_per_testcase(propagator_configuration_with_os_env, Config) -> application:set_env(opentelemetry, text_map_propagators, [b3multi, baggage]), {ok, _} = application:ensure_all_started(opentelemetry), Config; +init_per_testcase(new_propagator_configuration, Config) -> + application:set_env(opentelemetry, text_map_propagators, [jaeger]), + application:set_env(opentelemetry, propagator, #{composite => [b3multi, baggage], + composite_list => "baggage, tracecontext"}), + {ok, _} = application:ensure_all_started(opentelemetry), + Config; init_per_testcase(force_flush, Config) -> Config1 = set_batch_tab_processor(1000000, Config), {ok, _} = application:ensure_all_started(opentelemetry), @@ -329,6 +336,15 @@ propagator_configuration_with_os_env(_Config) -> ok. +new_propagator_configuration(_Config) -> + %% test that duplicate baggage entry from composite_list is removed in result + %% and old format of setting `text_map_propagators' is ignored + ?assertEqual({otel_propagator_text_map_composite, + [{otel_propagator_b3,b3multi},otel_propagator_baggage, otel_propagator_trace_context]}, opentelemetry:get_text_map_extractor()), + ?assertEqual({otel_propagator_text_map_composite, + [{otel_propagator_b3,b3multi},otel_propagator_baggage, otel_propagator_trace_context]}, opentelemetry:get_text_map_injector()), + ok. + force_flush(Config) -> Tid = ?config(tid, Config), SpanCtx1 = ?start_span(<<"span-1">>), diff --git a/apps/opentelemetry/test/otel_configuration_SUITE.erl b/apps/opentelemetry/test/otel_configuration_SUITE.erl index 704b208c..f71e4001 100644 --- a/apps/opentelemetry/test/otel_configuration_SUITE.erl +++ b/apps/opentelemetry/test/otel_configuration_SUITE.erl @@ -250,21 +250,21 @@ log_level(_Config) -> propagators(_Config) -> ?assertMatch(#{log_level := error, - text_map_propagators := [baggage]}, + propagator := #{composite := [baggage]}}, otel_configuration:merge_with_os([{log_level, error}])), ok. propagators_b3(_Config) -> ?assertMatch(#{log_level := error, - text_map_propagators := [b3]}, + propagator := #{composite := [b3]}}, otel_configuration:merge_with_os([{log_level, error}])), ok. propagators_b3multi(_Config) -> ?assertMatch(#{log_level := error, - text_map_propagators := [b3multi]}, + propagator := #{composite := [b3multi]}}, otel_configuration:merge_with_os([{log_level, error}])), ok. From 25538c4932edf607909aaf1fb87b6b5dfbc2d2ea Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Sun, 6 Jul 2025 04:24:42 -0400 Subject: [PATCH 06/15] ignore old attribute limits if new config key is set --- apps/opentelemetry/src/otel_configuration.erl | 37 +++++++++++-------- .../test/opentelemetry_SUITE.erl | 30 ++++++++++++++- 2 files changed, 49 insertions(+), 18 deletions(-) diff --git a/apps/opentelemetry/src/otel_configuration.erl b/apps/opentelemetry/src/otel_configuration.erl index 4bf1b0c1..4410681e 100644 --- a/apps/opentelemetry/src/otel_configuration.erl +++ b/apps/opentelemetry/src/otel_configuration.erl @@ -26,8 +26,8 @@ -type log_level() :: atom(). --type attribute_limits() :: #{attribute_value_length_limit := integer() | undefined, - attribute_count_limit := integer() | undefined +-type attribute_limits() :: #{attribute_value_length_limit => integer() | undefined, + attribute_count_limit => integer() | undefined }. -type exporter_args() :: map(). @@ -227,20 +227,25 @@ convert_disabled(OldConfig, Config) -> end. convert_attribute_limits(OldConfig, Config) -> - OldConfig2 = case maps:take(attribute_count_limit, OldConfig) of - {AttributeCountLimit, OldConfig1} -> - OldConfig1#{attribute_limits => #{attribute_count_limit => AttributeCountLimit}}; - error -> - OldConfig - end, - - - case maps:take(attribute_value_length_limit, OldConfig2) of - {AttributeValueLengthLimit, OldConfig3} -> - AttributeLimitsMap = maps:get(attribute_limits, OldConfig3, #{}), - OldConfig3#{attribute_limits => AttributeLimitsMap#{attribute_value_length_limit => AttributeValueLengthLimit}}; - error -> - OldConfig2 + case lists:keyfind(attribute_limits, 1, Config) of + {attribute_limits, Limits} -> + OldConfig#{attribute_limits => Limits}; + false -> + OldConfig2 = case maps:take(attribute_count_limit, OldConfig) of + {AttributeCountLimit, OldConfig1} -> + OldConfig1#{attribute_limits => #{attribute_count_limit => AttributeCountLimit}}; + error -> + OldConfig + end, + + + case maps:take(attribute_value_length_limit, OldConfig2) of + {AttributeValueLengthLimit, OldConfig3} -> + AttributeLimitsMap = maps:get(attribute_limits, OldConfig3, #{}), + OldConfig3#{attribute_limits => AttributeLimitsMap#{attribute_value_length_limit => AttributeValueLengthLimit}}; + error -> + OldConfig2 + end end. %% ConfigMap = new(), diff --git a/apps/opentelemetry/test/opentelemetry_SUITE.erl b/apps/opentelemetry/test/opentelemetry_SUITE.erl index a9373818..856ebfdd 100644 --- a/apps/opentelemetry/test/opentelemetry_SUITE.erl +++ b/apps/opentelemetry/test/opentelemetry_SUITE.erl @@ -40,7 +40,7 @@ all_cases() -> record_but_not_sample, record_exception_works, record_exception_with_message_works, propagator_configuration, propagator_configuration_with_os_env, force_flush, dropped_attributes, too_many_attributes, truncated_binary_attributes, - new_propagator_configuration]. + new_propagator_configuration, dropped_attributes_ignore_old_config]. groups() -> [{otel_simple_processor, [], all_cases()}, @@ -117,6 +117,16 @@ init_per_testcase(dropped_attributes, Config) -> Config1 = set_batch_tab_processor(Config), application:set_env(opentelemetry, attribute_value_length_limit, 2), + + {ok, _} = application:ensure_all_started(opentelemetry), + + Config1; +init_per_testcase(dropped_attributes_ignore_old_config, Config) -> + Config1 = set_batch_tab_processor(Config), + + application:set_env(opentelemetry, attribute_value_length_limit, 500), + application:set_env(opentelemetry, attribute_limits, #{attribute_value_length_limit => 2, + attribute_count_limit => 1}), {ok, _} = application:ensure_all_started(opentelemetry), Config1; @@ -169,6 +179,7 @@ end_per_testcase(propagator_configuration_with_os_env, _Config) -> ok; end_per_testcase(_, _Config) -> application:unset_env(opentelemetry, attribute_value_length_limit), + application:unset_env(opentelemetry, attribute_limits), _ = application:stop(opentelemetry), ok. @@ -453,7 +464,6 @@ macros(Config) -> otel_span:end_span(SpanCtx1), [Span1] = assert_exported(Tid, SpanCtx1), - ?assertEqual(#{Attr1 => AttrValue1}, otel_attributes:map(Span1#span.attributes)), ok. @@ -1017,6 +1027,22 @@ dropped_attributes(Config) -> ok. +dropped_attributes_ignore_old_config(Config) -> + Tid = ?config(tid, Config), + SpanCtx = ?start_span(<<"span-1">>), + + ?set_current_span(SpanCtx), + + ?set_attribute(<<"attr-1">>, <<"attr-value-1">>), + ?set_attribute(<<"attr-2">>, {non_homogeneous, <<"attribute">>}), + + otel_span:end_span(SpanCtx), + [Span] = assert_exported(Tid, SpanCtx), + + ?assertEqual(#{<<"attr-1">> => <<"at">>}, otel_attributes:map(Span#span.attributes)), + + ok. + truncated_binary_attributes(_Config) -> InfinityLengthAttributes = otel_attributes:new(#{<<"attr-1">> => <<"abcde">>, <<"attr-2">> => [<<"a">>, <<"abcde">>, <<"abcde">>]}, From 50d94f822c99d483fdbf0cb9f5b36ff77d155b4f Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Mon, 7 Jul 2025 05:58:16 -0400 Subject: [PATCH 07/15] convert processor options to new form --- .../src/otel_batch_processor.erl | 4 +- apps/opentelemetry/src/otel_configuration.erl | 78 +++++++++++++++++-- apps/opentelemetry/src/otel_tracer_server.erl | 12 ++- .../src/otel_tracer_server_sup.erl | 4 +- .../test/opentelemetry_SUITE.erl | 17 ++-- .../test/otel_configuration_SUITE.erl | 36 +++++---- 6 files changed, 113 insertions(+), 38 deletions(-) diff --git a/apps/opentelemetry/src/otel_batch_processor.erl b/apps/opentelemetry/src/otel_batch_processor.erl index dca06d71..d1c78468 100644 --- a/apps/opentelemetry/src/otel_batch_processor.erl +++ b/apps/opentelemetry/src/otel_batch_processor.erl @@ -147,8 +147,8 @@ init([Args=#{reg_name := RegName}]) -> process_flag(trap_exit, true), SizeLimit = maps:get(max_queue_size, Args, ?DEFAULT_MAX_QUEUE_SIZE), - ExportingTimeout = maps:get(exporting_timeout_ms, Args, ?DEFAULT_EXPORTER_TIMEOUT_MS), - ScheduledDelay = maps:get(scheduled_delay_ms, Args, ?DEFAULT_SCHEDULED_DELAY_MS), + ExportingTimeout = maps:get(export_timeout, Args, ?DEFAULT_EXPORTER_TIMEOUT_MS), + ScheduledDelay = maps:get(schedule_delay, Args, ?DEFAULT_SCHEDULED_DELAY_MS), CheckTableSize = maps:get(check_table_size_ms, Args, ?DEFAULT_CHECK_TABLE_SIZE_MS), %% TODO: this should be passed in from the tracer server diff --git a/apps/opentelemetry/src/otel_configuration.erl b/apps/opentelemetry/src/otel_configuration.erl index 4410681e..19eb5a76 100644 --- a/apps/opentelemetry/src/otel_configuration.erl +++ b/apps/opentelemetry/src/otel_configuration.erl @@ -44,7 +44,10 @@ -type span_processor_type() :: batch | simple | atom(). --type span_processor() :: #{span_processor_type() => batch_span_processor() | simple_span_processor() | otel_config_properties:t()}. +-type span_processor() :: batch_span_processor() | + simple_span_processor() | + otel_config_properties:t() +-type span_processors() :: #{span_processor_type() => span_processor()}. -type propagator() :: tracecontext | baggage | b3 | b3multi | jaeger | ottrace | atom(). @@ -54,7 +57,7 @@ attribute_limits => attribute_limits(), propagator := #{composite => [propagator()], composite_list => string()}, - tracer_provider => #{processors => [span_processor()], + tracer_provider => #{processors => [span_processors()], limits => #{}, sampler => #{}} }. @@ -164,10 +167,73 @@ merge_with_os(Config) -> -spec convert_to_new(#{}, map()) -> t(). convert_to_new(OldConfig, Config) -> - convert_propagator( - convert_resource( - convert_attribute_limits( - convert_disabled(OldConfig, Config), Config), Config), Config). + convert_tracer_provider( + convert_propagator( + convert_resource( + convert_attribute_limits( + convert_disabled(OldConfig, Config), Config), Config), Config), Config). + +convert_tracer_provider(OldConfig, Config) -> + %% bsp_scheduled_delay_ms := integer() | undefined, + %% bsp_exporting_timeout_ms := integer() | undefined, + %% bsp_max_queue_size := integer() | undefined, + %% ssp_exporting_timeout_ms := integer() | undefined, + %% traces_exporter := {atom(), term()} | none | undefined, + %% processors := list(), + %% sampler := {atom(), term()}, + case lists:keyfind(tracer_provider, 1, Config) of + {tracer_provider, TracerProvider} -> + OldConfig#{tracer_provider => TracerProvider}; + false -> + case maps:take(processors, OldConfig) of + error -> + OldConfig; + {Processors, OldConfig1} -> + NewProcessors = lists:map(fun({otel_batch_processor, BatchConfig}) -> + {batch, convert_batch(BatchConfig)}; + ({otel_simple_processor, SimpleConfig}) -> + {simple, convert_simple(SimpleConfig)} + end, Processors), + OldConfig1#{tracer_provider => #{processors => NewProcessors}} + end + end. + +convert_simple(SimpleConfig) -> + SimpleConfig1 = update_processor_config(SimpleConfig, [{exporting_timeout_ms, export_timeout}]), + + case maps:take(exporter, SimpleConfig1) of + {ExporterConfig, SimpleConfig2} -> + SimpleConfig2#{exporter => convert_exporter(ExporterConfig)}; + error -> + SimpleConfig1 + end. + +convert_batch(BatchConfig) -> + BatchConfig1 = update_processor_config(BatchConfig, [{scheduled_delay_ms, schedule_delay}, + {exporting_timeout_ms, export_timeout} + %% {max_queue_size, max_queue_size}, + %% {max_export_batch_size}, + ]), + + case maps:take(exporter, BatchConfig1) of + error -> + BatchConfig1; + {ExporterValue, BatchConfig2} -> + BatchConfig2#{exporter => convert_exporter(ExporterValue)} + end. + +update_processor_config(Config, Options) -> + lists:foldl(fun({OldKey, NewKey}, ConfigAcc) -> + case maps:take(OldKey, ConfigAcc) of + error -> + ConfigAcc; + {Value, ConfigAcc1} -> + ConfigAcc1#{NewKey => Value} + end + end, Config, Options). + +convert_exporter(ExporterConfig) -> + ExporterConfig. %% convert the old `text_map_propagators' config into the new `propagator' config %% based on the declarative file configuration of otel diff --git a/apps/opentelemetry/src/otel_tracer_server.erl b/apps/opentelemetry/src/otel_tracer_server.erl index bf5f30f3..46787e8e 100644 --- a/apps/opentelemetry/src/otel_tracer_server.erl +++ b/apps/opentelemetry/src/otel_tracer_server.erl @@ -61,9 +61,11 @@ start_link(Name, RegName, SpanProcessorSupRegName, Resource, Config) -> init([Name, SpanProcessorSup, Resource, Config]) -> IdGeneratorModule = maps:get(id_generator, Config, otel_id_generator), SamplerSpec = maps:get(sampler, Config, {parent_based, #{root => always_on}}), - Processors = maps:get(processors, Config, [{otel_batch_processor, #{}}]), DenyList = maps:get(deny_list, Config, []), + TracerProviderConfig = maps:get(tracer_provider, Config, #{}), + Processors = maps:get(processors, TracerProviderConfig, [{otel_batch_processor, #{}}]), + Sampler = otel_sampler:new(SamplerSpec), Processors1 = init_processors(SpanProcessorSup, Processors), @@ -136,8 +138,8 @@ update_force_flush_error(Reason, {error, List}) -> %% is only one, like was once guaranteed. This allows functions like `set_exporter' %% to work. But if there is more than one processor defined or it isn't one of the %% builtin processors we will not do this hack of adding a name to the config. -init_processors(SpanProcessorSup, [{P, Config}]) when P =:= otel_batch_processor ; - P =:= otel_simple_processor -> +init_processors(SpanProcessorSup, [{P, Config}]) when P =:= batch ; + P =:= simple -> case init_processor(SpanProcessorSup, P, maps:merge(#{name => global}, Config)) of {true, {_, _}=Processor} -> [Processor]; @@ -157,6 +159,10 @@ init_processors_(SpanProcessorSup, [{P, Config} | Rest]) -> init_processors_(SpanProcessorSup, Rest) end. +init_processor(SpanProcessorSup, batch, Config) -> + init_processor(SpanProcessorSup, otel_batch_processor, Config); +init_processor(SpanProcessorSup, simple, Config) -> + init_processor(SpanProcessorSup, otel_simple_processor, Config); init_processor(SpanProcessorSup, ProcessorModule, Config) -> %% start_link is an optional callback for processors case lists:member({start_link, 1}, ProcessorModule:module_info(exports)) of diff --git a/apps/opentelemetry/src/otel_tracer_server_sup.erl b/apps/opentelemetry/src/otel_tracer_server_sup.erl index e262e892..ff62e74b 100644 --- a/apps/opentelemetry/src/otel_tracer_server_sup.erl +++ b/apps/opentelemetry/src/otel_tracer_server_sup.erl @@ -36,7 +36,7 @@ init([Name, Resource, Opts]) -> %% supervisors and the tracer provider itself. %% in the case of the global provider the name is `global' SpanProcessorSupRegName = list_to_atom(lists:concat([otel_span_processor_sup, "_", Name])), - TracerServeRegName = list_to_atom(lists:concat([otel_tracer_provider, "_", Name])), + TracerServerRegName = list_to_atom(lists:concat([otel_tracer_provider, "_", Name])), SpanProcessorSup = #{id => otel_span_processor_sup, start => {otel_span_processor_sup, start_link, [SpanProcessorSupRegName]}, @@ -47,7 +47,7 @@ init([Name, Resource, Opts]) -> TracerServer = #{id => otel_tracer_server, start => {otel_tracer_server, start_link, [Name, - TracerServeRegName, + TracerServerRegName, SpanProcessorSupRegName, Resource, Opts]}, diff --git a/apps/opentelemetry/test/opentelemetry_SUITE.erl b/apps/opentelemetry/test/opentelemetry_SUITE.erl index 856ebfdd..4d8cbeaf 100644 --- a/apps/opentelemetry/test/opentelemetry_SUITE.erl +++ b/apps/opentelemetry/test/opentelemetry_SUITE.erl @@ -671,9 +671,9 @@ multiple_tracer_providers(_Config) -> Resource, #{id_generator => otel_id_generator, sampler => {otel_sampler_always_on, []}, - processors => [{otel_batch_processor, #{name => test_batch, - scheduled_delay_ms => 1, - exporter => {otel_exporter_pid, self()}}}], + tracer_provider => #{processors => [{batch, #{name => test_batch, + schedule_delay => 1, + exporter => {otel_exporter_pid, self()}}}]}, deny_list => []})), ?assertEqual(Resource, otel_tracer_provider:resource(test_provider)), @@ -681,9 +681,10 @@ multiple_tracer_providers(_Config) -> ?assertMatch({ok, _}, opentelemetry:start_tracer_provider(deprecated_test_provider_start, #{id_generator => otel_id_generator, sampler => {otel_sampler_always_on, []}, - processors => [{otel_batch_processor, #{name => test_batch_2, - scheduled_delay_ms => 1000, - exporter => {otel_exporter_pid, self()}}}], + tracer_provider => + #{processors => [{batch, #{name => test_batch_2, + schedule_delay => 1, + exporter => {otel_exporter_pid, self()}}}]}, deny_list => []})), ?assertEqual(otel_resource:create([]), otel_tracer_provider:resource(deprecated_test_provider_start)), @@ -707,7 +708,7 @@ multiple_tracer_providers(_Config) -> ?assertEqual(<<"span-1">>, Span#span.name) after 1000 -> - ct:fail(failed) + ct:fail(failed1) end, %% now a span with the tracer from the non-global tracer provider @@ -723,7 +724,7 @@ multiple_tracer_providers(_Config) -> ?assertEqual(<<"span-2">>, Span1#span.name) after 1000 -> - ct:fail(failed) + ct:fail(failed2) end, ok. diff --git a/apps/opentelemetry/test/otel_configuration_SUITE.erl b/apps/opentelemetry/test/otel_configuration_SUITE.erl index f71e4001..f0805615 100644 --- a/apps/opentelemetry/test/otel_configuration_SUITE.erl +++ b/apps/opentelemetry/test/otel_configuration_SUITE.erl @@ -289,10 +289,10 @@ none_exporter(_Config) -> ?assertMatch(none, maps:get(traces_exporter, otel_configuration:merge_with_os([]))), - ?assertMatch(#{processors := [{otel_simple_processor, #{exporter := none}}]}, + ?assertMatch(#{tracer_provider := #{processors := [{simple, #{exporter := none}}]}}, otel_configuration:merge_with_os([{span_processor, simple}])), - ?assertMatch(#{processors := [{otel_batch_processor, #{exporter := none}}]}, + ?assertMatch(#{tracer_provider := #{processors := [{batch, #{exporter := none}}]}}, otel_configuration:merge_with_os([{span_processor, batch}])), ok. @@ -370,37 +370,39 @@ compare_span_limits(Config) -> ok. span_processors(_Config) -> - ?assertMatch(#{processors := [{otel_simple_processor, #{}}]}, + ?assertMatch(#{tracer_provider := #{processors := [{simple, #{}}]}}, otel_configuration:merge_with_os([{span_processor, simple}])), - ?assertMatch(#{processors := [{otel_batch_processor, #{exporting_timeout_ms := 2, - max_queue_size := 1, - scheduled_delay_ms := 15000}}]}, + ?assertMatch(#{tracer_provider := + #{processors := [{batch, #{export_timeout := 2, + max_queue_size := 1, + schedule_delay := 15000}}]}}, otel_configuration:merge_with_os([{span_processor, batch}, {bsp_scheduled_delay_ms, 15000}, {bsp_exporting_timeout_ms, 2}, {bsp_max_queue_size, 1}])), - ?assertMatch(#{processors := [{otel_batch_processor, #{exporter := none}}]}, + ?assertMatch(#{tracer_provider := #{processors := [{batch, #{exporter := none}}]}}, otel_configuration:merge_with_os([{span_processor, batch}, {traces_exporter, none}])), - ?assertMatch(#{processors := [{otel_batch_processor, #{}}]}, + ?assertMatch(#{tracer_provider := #{processors := [{batch, #{}}]}}, otel_configuration:merge_with_os([{span_processor, batch}])), - ?assertMatch(#{processors := [{otel_batch_processor, #{exporter := {opentelemetry_exporter, - #{endpoints := ["https://example.com"]}}, - exporting_timeout_ms := 2, - max_queue_size := 1, - scheduled_delay_ms := 15000}}]}, - otel_configuration:merge_with_os([{processors, [{otel_batch_processor, #{exporter => {opentelemetry_exporter,#{endpoints => ["https://example.com"]}}, - scheduled_delay_ms => 15000, + ?assertMatch(#{tracer_provider := + #{processors := [{batch, #{exporter := {opentelemetry_exporter, + #{endpoints := ["https://example.com"]}}, + export_timeout := 2, + max_queue_size := 1, + schedule_delay := 15000}}]}}, + otel_configuration:merge_with_os([{processors, [{otel_batch_processor, #{exporter => {opentelemetry_exporter, #{endpoints => ["https://example.com"]}}, + schedule_delay => 15000, max_queue_size => 4, - exporting_timeout_ms => 3}}]}, + export_timeout => 3}}]}, {bsp_exporting_timeout_ms, 2}, {bsp_max_queue_size, 1}])), - ?assertMatch(#{processors := [{otel_simple_processor, #{exporting_timeout_ms := 2}}]}, + ?assertMatch(#{tracer_provider := #{processors := [{simple, #{export_timeout := 2}}]}}, otel_configuration:merge_with_os([{span_processor, simple}, {ssp_exporting_timeout_ms, 2}])), From 6bef942d5fd148df2b1fbc7227adbbd8b8042e57 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Sat, 12 Jul 2025 06:41:57 -0400 Subject: [PATCH 08/15] convert tracer provider config to new form --- apps/opentelemetry/src/otel_configuration.erl | 57 ++++++++++++++++++- .../test/otel_configuration_SUITE.erl | 10 +++- 2 files changed, 62 insertions(+), 5 deletions(-) diff --git a/apps/opentelemetry/src/otel_configuration.erl b/apps/opentelemetry/src/otel_configuration.erl index 19eb5a76..7de8dac9 100644 --- a/apps/opentelemetry/src/otel_configuration.erl +++ b/apps/opentelemetry/src/otel_configuration.erl @@ -46,11 +46,37 @@ -type span_processor() :: batch_span_processor() | simple_span_processor() | - otel_config_properties:t() + otel_config_properties:t(). -type span_processors() :: #{span_processor_type() => span_processor()}. -type propagator() :: tracecontext | baggage | b3 | b3multi | jaeger | ottrace | atom(). +-type limits() :: #{%% Configure max attribute value size. Overrides .attribute_limits.attribute_value_length_limit. + %% Value must be non-negative. + %% If omitted or undefined, there is no limit. + attribute_value_length_limit => integer() | undefined, + %% Configure max attribute count. Overrides .attribute_limits.attribute_count_limit. + %% Value must be non-negative. + %% If omitted or undefined, 128 is used. + attribute_count_limit => integer() | undefined, + %% Configure max span event count. + %% Value must be non-negative. + %% If omitted or undefined, 128 is used. + event_count_limit => integer() | undefined, + %% Configure max span link count. + %% Value must be non-negative. + %% If omitted or undefined, 128 is used. + link_count_limit => integer() | undefined, + %% Configure max attributes per span event. + %% Value must be non-negative. + %% If omitted or undefined, 128 is used. + event_attribute_count_limit => integer() | undefined, + %% Configure max attributes per span link. + %% Value must be non-negative. + %% If omitted or undefined, 128 is used. + link_attribute_count_limit => integer() | undefined + }. + -type t() :: #{disabled => boolean(), log_level => log_level(), resource => #{attributes => opentelemetry:attributes_map()}, @@ -58,7 +84,7 @@ propagator := #{composite => [propagator()], composite_list => string()}, tracer_provider => #{processors => [span_processors()], - limits => #{}, + limits => limits(), sampler => #{}} }. @@ -187,6 +213,7 @@ convert_tracer_provider(OldConfig, Config) -> false -> case maps:take(processors, OldConfig) of error -> + TracerProvider = convert_span_limits(OldConfig), OldConfig; {Processors, OldConfig1} -> NewProcessors = lists:map(fun({otel_batch_processor, BatchConfig}) -> @@ -194,10 +221,34 @@ convert_tracer_provider(OldConfig, Config) -> ({otel_simple_processor, SimpleConfig}) -> {simple, convert_simple(SimpleConfig)} end, Processors), - OldConfig1#{tracer_provider => #{processors => NewProcessors}} + + TracerProvider = convert_span_limits(OldConfig), + + OldConfig1#{tracer_provider => TracerProvider#{processors => NewProcessors}} end end. +convert_span_limits(OldConfig) -> + AttributeLimits = maps:get(attribute_limits, OldConfig, #{}), + #{limits => new_map_updated_keys(maps:merge(OldConfig, AttributeLimits), + [{attribute_count_limit, attribute_count_limit}, + {attribute_value_length_limit, attribute_value_length_limit}, + {event_count_limit, event_count_limit}, + {link_count_limit, link_count_limit}, + {attribute_per_event_limit, event_attribute_count_limit}, + {attribute_per_link_limit, link_attribute_count_limit}])}. + +new_map_updated_keys(Config, Options) -> + lists:foldl(fun({OldKey, NewKey}, ConfigAcc) -> + case maps:find(OldKey, Config) of + error -> + ConfigAcc; + {ok, Value} -> + ConfigAcc#{NewKey => Value} + end + end, #{}, Options). + + convert_simple(SimpleConfig) -> SimpleConfig1 = update_processor_config(SimpleConfig, [{exporting_timeout_ms, export_timeout}]), diff --git a/apps/opentelemetry/test/otel_configuration_SUITE.erl b/apps/opentelemetry/test/otel_configuration_SUITE.erl index f0805615..6c1e77fd 100644 --- a/apps/opentelemetry/test/otel_configuration_SUITE.erl +++ b/apps/opentelemetry/test/otel_configuration_SUITE.erl @@ -370,8 +370,14 @@ compare_span_limits(Config) -> ok. span_processors(_Config) -> - ?assertMatch(#{tracer_provider := #{processors := [{simple, #{}}]}}, - otel_configuration:merge_with_os([{span_processor, simple}])), + ?assertMatch(#{tracer_provider := #{processors := [{simple, #{}}], + limits := #{attribute_value_length_limit := 10, + event_attribute_count_limit := 400, + event_count_limit := 20}}}, + otel_configuration:merge_with_os([{span_processor, simple}, + {attribute_value_length_limit, 10}, + {attribute_per_event_limit, 400}, + {event_count_limit, 20}])), ?assertMatch(#{tracer_provider := #{processors := [{batch, #{export_timeout := 2, From e18dc0def0d78a7bfa14083a18728efac30f9e7a Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Sun, 13 Jul 2025 06:42:44 -0400 Subject: [PATCH 09/15] convert sampler config to be under tracer_provider --- apps/opentelemetry/src/otel_configuration.erl | 97 +++++++++++-------- apps/opentelemetry/src/otel_sampler.erl | 41 +++++--- .../src/otel_sampler_parent_based.erl | 27 ++++-- apps/opentelemetry/src/otel_tracer_server.erl | 7 +- .../test/opentelemetry_SUITE.erl | 29 +++++- .../test/otel_configuration_SUITE.erl | 38 +++++--- 6 files changed, 160 insertions(+), 79 deletions(-) diff --git a/apps/opentelemetry/src/otel_configuration.erl b/apps/opentelemetry/src/otel_configuration.erl index 7de8dac9..74540f49 100644 --- a/apps/opentelemetry/src/otel_configuration.erl +++ b/apps/opentelemetry/src/otel_configuration.erl @@ -19,7 +19,8 @@ %%%------------------------------------------------------------------------- -module(otel_configuration). --export([merge_with_os/1, +-export([new/0, + merge_with_os/1, merge_list_with_environment/3, transform/2, report_cb/1]). @@ -77,6 +78,16 @@ link_attribute_count_limit => integer() | undefined }. +-type sampler() :: {always_on, #{}} + | {always_off, #{}} + | {trace_id_ratio_based, #{ratio => float()}} + | {parent_based, #{remote_parent_sampled => sampler(), + remote_parent_not_sampled => sampler(), + local_parent_sampled => sampler(), + local_parent_not_sampled => sampler(), + root => sampler()}} + | {atom(), otel_config_properties:t()}. + -type t() :: #{disabled => boolean(), log_level => log_level(), resource => #{attributes => opentelemetry:attributes_map()}, @@ -85,7 +96,7 @@ composite_list => string()}, tracer_provider => #{processors => [span_processors()], limits => limits(), - sampler => #{}} + sampler => sampler()} }. %% structure before the standard otel declarative configuration @@ -213,8 +224,9 @@ convert_tracer_provider(OldConfig, Config) -> false -> case maps:take(processors, OldConfig) of error -> - TracerProvider = convert_span_limits(OldConfig), - OldConfig; + {TracerProvider, OldConfig1} = convert_sampler(OldConfig), + TracerProvider1 = convert_span_limits(TracerProvider, OldConfig1), + OldConfig1#{tracer_provider => maps:merge(TracerProvider1)}; {Processors, OldConfig1} -> NewProcessors = lists:map(fun({otel_batch_processor, BatchConfig}) -> {batch, convert_batch(BatchConfig)}; @@ -222,21 +234,30 @@ convert_tracer_provider(OldConfig, Config) -> {simple, convert_simple(SimpleConfig)} end, Processors), - TracerProvider = convert_span_limits(OldConfig), + {TracerProvider, OldConfig2} = convert_sampler(OldConfig1), + TracerProvider1 = convert_span_limits(TracerProvider, OldConfig2), - OldConfig1#{tracer_provider => TracerProvider#{processors => NewProcessors}} + OldConfig2#{tracer_provider => TracerProvider1#{processors => NewProcessors}} end end. -convert_span_limits(OldConfig) -> +convert_sampler(OldConfig) -> + case maps:take(sampler, OldConfig) of + {Sampler, OldConfig1} -> + {#{sampler => Sampler}, OldConfig1}; + error -> + {#{}, OldConfig} + end. + +convert_span_limits(TracerProvider, OldConfig) -> AttributeLimits = maps:get(attribute_limits, OldConfig, #{}), - #{limits => new_map_updated_keys(maps:merge(OldConfig, AttributeLimits), - [{attribute_count_limit, attribute_count_limit}, - {attribute_value_length_limit, attribute_value_length_limit}, - {event_count_limit, event_count_limit}, - {link_count_limit, link_count_limit}, - {attribute_per_event_limit, event_attribute_count_limit}, - {attribute_per_link_limit, link_attribute_count_limit}])}. + TracerProvider#{limits => new_map_updated_keys(maps:merge(OldConfig, AttributeLimits), + [{attribute_count_limit, attribute_count_limit}, + {attribute_value_length_limit, attribute_value_length_limit}, + {event_count_limit, event_count_limit}, + {link_count_limit, link_count_limit}, + {attribute_per_event_limit, event_attribute_count_limit}, + {attribute_per_link_limit, link_attribute_count_limit}])}. new_map_updated_keys(Config, Options) -> lists:foldl(fun({OldKey, NewKey}, ConfigAcc) -> @@ -377,25 +398,25 @@ convert_attribute_limits(OldConfig, Config) -> --spec resource(list(), t()) -> t(). -resource(AppEnv, ConfigMap) -> - Resource = proplists:get_value(resource, AppEnv, []), - ConfigMap. +%% -spec resource(list(), t()) -> t(). +%% resource(AppEnv, ConfigMap) -> +%% Resource = proplists:get_value(resource, AppEnv, []), +%% ConfigMap. --spec attribute_limits(list(), t()) -> t(). -attribute_limits(AppEnv, ConfigMap) -> - AttributeLimits = proplists:get_value(attribute_limits, AppEnv, []), - ConfigMap. +%% -spec attribute_limits(list(), t()) -> t(). +%% attribute_limits(AppEnv, ConfigMap) -> +%% AttributeLimits = proplists:get_value(attribute_limits, AppEnv, []), +%% ConfigMap. --spec propagator(list(), t()) -> t(). -propagator(AppEnv, ConfigMap) -> - Propagator = proplists:get_value(propagator, AppEnv, []), - ConfigMap. +%% -spec propagator(list(), t()) -> t(). +%% propagator(AppEnv, ConfigMap) -> +%% Propagator = proplists:get_value(propagator, AppEnv, []), +%% ConfigMap. --spec tracer_provider(list(), t()) -> t(). -tracer_provider(AppEnv, ConfigMap) -> - TracerProvider = proplists:get_value(tracer_provider, AppEnv, []), - ConfigMap. +%% -spec tracer_provider(list(), t()) -> t(). +%% tracer_provider(AppEnv, ConfigMap) -> +%% TracerProvider = proplists:get_value(tracer_provider, AppEnv, []), +%% ConfigMap. %% backwards compatability @@ -698,21 +719,21 @@ transform(url, Value) -> uri_string:parse(Value); %% convert sampler string to usable configuration term transform(sampler, {"parentbased_always_on", _}) -> - {parent_based, #{root => always_on}}; + {parent_based, #{root => {always_on, #{}}}}; transform(sampler, {"parentbased_always_off", _}) -> - {parent_based, #{root => always_off}}; + {parent_based, #{root => {always_off, #{}}}}; transform(sampler, {"always_on", _}) -> - always_on; + {always_on, #{}}; transform(sampler, {"always_off", _}) -> - always_off; + {always_off, #{}}; transform(sampler, {"traceidratio", false}) -> - {trace_id_ratio_based, 1.0}; + {trace_id_ratio_based, #{ratio => 1.0}}; transform(sampler, {"traceidratio", Probability}) -> - {trace_id_ratio_based, probability_string_to_float(Probability)}; + {trace_id_ratio_based, #{ratio => probability_string_to_float(Probability)}}; transform(sampler, {"parentbased_traceidratio", false}) -> - {parent_based, #{root => {trace_id_ratio_based, 1.0}}}; + {parent_based, #{root => {trace_id_ratio_based, #{ratio => 1.0}}}}; transform(sampler, {"parentbased_traceidratio", Probability}) -> - {parent_based, #{root => {trace_id_ratio_based, probability_string_to_float(Probability)}}}; + {parent_based, #{root => {trace_id_ratio_based, #{ratio => probability_string_to_float(Probability)}}}}; transform(sampler, Value) -> Value; transform(key_value_list, Value) when is_list(Value) -> diff --git a/apps/opentelemetry/src/otel_sampler.erl b/apps/opentelemetry/src/otel_sampler.erl index 39f2aa75..8914ff08 100644 --- a/apps/opentelemetry/src/otel_sampler.erl +++ b/apps/opentelemetry/src/otel_sampler.erl @@ -33,7 +33,10 @@ %%%------------------------------------------------------------------------- -module(otel_sampler). --export([description/1, new/1, should_sample/7]). +-export([description/1, + new/1, + new/2, + should_sample/7]). -export_type([description/0, sampler_spec/0, @@ -73,19 +76,13 @@ %% Any options passed to a sampler. -type builtin_sampler() :: - always_on - | always_off - | {trace_id_ratio_based, float()} - | {parent_based, #{ - remote_parent_sampled => sampler_spec(), - remote_parent_not_sampled => sampler_spec(), - local_parent_sampled => sampler_spec(), - local_parent_not_sampled => sampler_spec(), - root => sampler_spec() - }}. + always_on + | always_off + | trace_id_ratio_based + | parent_based. %% A built-in sampler. --type sampler_spec() :: builtin_sampler() | {module(), sampler_opts()}. +-type sampler_spec() :: {builtin_sampler() | atom(), sampler_opts()}. %% Specification to create a sampler. -type sampling_decision() :: ?DROP | ?RECORD_ONLY | ?RECORD_AND_SAMPLE. @@ -101,12 +98,17 @@ -opaque t() :: {module(), description(), sampler_opts()}. %% A sampler. -%% @doc Returns a sampler based on the given specification. +%% @doc Returns a sampler based on the given specification. This is left for backwards +%% compatibility. The function `new/2' should be used instead. -spec new(SamplerSpec :: sampler_spec()) -> t(). new(always_on) -> new({otel_sampler_always_on, #{}}); new(always_off) -> new({otel_sampler_always_off, #{}}); +new({always_on, Opts}) -> + new({otel_sampler_always_on, Opts}); +new({always_off, Opts}) -> + new({otel_sampler_always_off, Opts}); new({trace_id_ratio_based, Opts}) -> new({otel_sampler_trace_id_ratio_based, Opts}); new({parent_based, Opts}) -> @@ -115,6 +117,19 @@ new({Sampler, Opts}) -> Config = Sampler:setup(Opts), {Sampler, Sampler:description(Config), Config}. +-spec new(builtin_sampler() | atom(), sampler_opts()) -> t(). +new(always_on, Opts) -> + new(otel_sampler_always_on, Opts); +new(always_off, Opts) -> + new(otel_sampler_always_off, Opts); +new(trace_id_ratio_based, Opts) -> + new(otel_sampler_trace_id_ratio_based, Opts); +new(parent_based, Opts) -> + new(otel_sampler_parent_based, Opts); +new(Sampler, Opts) -> + Config = Sampler:setup(Opts), + {Sampler, Sampler:description(Config), Config}. + %% @private -spec should_sample( t(), diff --git a/apps/opentelemetry/src/otel_sampler_parent_based.erl b/apps/opentelemetry/src/otel_sampler_parent_based.erl index 3e2899b2..ee13b9ba 100644 --- a/apps/opentelemetry/src/otel_sampler_parent_based.erl +++ b/apps/opentelemetry/src/otel_sampler_parent_based.erl @@ -52,12 +52,12 @@ %% Options to configure this sampler. %% @private -setup(Opts = #{root := RootSpec}) -> - RemoteParentSampledSampler = sampler_for_spec(remote_parent_sampled, Opts, always_on), - RemoteParentNotSampledSampler = sampler_for_spec(remote_parent_not_sampled, Opts, always_off), - LocalParentSampledSampler = sampler_for_spec(local_parent_sampled, Opts, always_on), - LocalParentNotSampledSampler = sampler_for_spec(local_parent_not_sampled, Opts, always_off), - RootSampler = otel_sampler:new(RootSpec), +setup(Opts = #{root := {RootName, RootSpec}}) -> + RemoteParentSampledSampler = sampler_for_spec(remote_parent_sampled, Opts, {always_on, #{}}), + RemoteParentNotSampledSampler = sampler_for_spec(remote_parent_not_sampled, Opts, {always_off, #{}}), + LocalParentSampledSampler = sampler_for_spec(local_parent_sampled, Opts, {always_on, #{}}), + LocalParentNotSampledSampler = sampler_for_spec(local_parent_not_sampled, Opts, {always_off, #{}}), + RootSampler = otel_sampler:new(RootName, RootSpec), #{ root => RootSampler, remote_parent_sampled => RemoteParentSampledSampler, @@ -67,11 +67,18 @@ setup(Opts = #{root := RootSpec}) -> }; setup(Opts) -> ?LOG_INFO("No sampler spec found for parent_based 'root' option. The 'always_on' sampler will be used for root spans."), - setup(Opts#{root => always_on}). + setup(Opts#{root => {always_on, #{}}}). -sampler_for_spec(Key, Opts, DefaultModule) -> - Spec = maps:get(Key, Opts, DefaultModule), - otel_sampler:new(Spec). +sampler_for_spec(Key, Opts, DefaultSampler) -> + case maps:get(Key, Opts, DefaultSampler) of + {Name, Opts} -> + otel_sampler:new(Name, Opts); + %% backwards compatibility: we used to accept a solo name of the sampler for those that + %% don't currently take arguments, like `always_on' but have now standardized on all + %% samplers being defined with options + Name -> + otel_sampler:new(Name) + end. %% @private description(#{ diff --git a/apps/opentelemetry/src/otel_tracer_server.erl b/apps/opentelemetry/src/otel_tracer_server.erl index 46787e8e..4bfe2863 100644 --- a/apps/opentelemetry/src/otel_tracer_server.erl +++ b/apps/opentelemetry/src/otel_tracer_server.erl @@ -59,11 +59,12 @@ start_link(Name, RegName, SpanProcessorSupRegName, Resource, Config) -> gen_server:start_link({local, RegName}, ?MODULE, [Name, SpanProcessorSupRegName, Resource, Config], []). init([Name, SpanProcessorSup, Resource, Config]) -> - IdGeneratorModule = maps:get(id_generator, Config, otel_id_generator), - SamplerSpec = maps:get(sampler, Config, {parent_based, #{root => always_on}}), + TracerProviderConfig = maps:get(tracer_provider, Config, #{}), + IdGeneratorModule = maps:get(id_generator, TracerProviderConfig, otel_id_generator), + SamplerSpec = maps:get(sampler, TracerProviderConfig, {parent_based, #{root => {always_on, #{}}}}), + ct:pal("Sampler ~p", [SamplerSpec]), DenyList = maps:get(deny_list, Config, []), - TracerProviderConfig = maps:get(tracer_provider, Config, #{}), Processors = maps:get(processors, TracerProviderConfig, [{otel_batch_processor, #{}}]), Sampler = otel_sampler:new(SamplerSpec), diff --git a/apps/opentelemetry/test/opentelemetry_SUITE.erl b/apps/opentelemetry/test/opentelemetry_SUITE.erl index 4d8cbeaf..940fbcb6 100644 --- a/apps/opentelemetry/test/opentelemetry_SUITE.erl +++ b/apps/opentelemetry/test/opentelemetry_SUITE.erl @@ -33,7 +33,7 @@ all() -> {group, otel_batch_processor}]. all_cases() -> - [with_span, macros, child_spans, disabled_sdk, + [with_span, macros, child_spans, disabled_sdk, always_off, update_span_data, tracer_instrumentation_scope, tracer_previous_ctx, stop_temporary_app, reset_after, attach_ctx, default_sampler, non_recording_ets_table, root_span_sampling_always_on, root_span_sampling_always_off, @@ -77,6 +77,11 @@ init_per_testcase(old_disable_auto_creation, Config) -> application:set_env(opentelemetry, register_loaded_applications, false), {ok, _} = application:ensure_all_started(opentelemetry), Config; +init_per_testcase(always_off, Config) -> + application:set_env(opentelemetry, sampler, always_off), + {ok, _} = application:ensure_all_started(opentelemetry), + Config1 = set_batch_tab_processor(1000000, Config), + Config1; init_per_testcase(application_tracers, Config) -> %% if both are set then the new one, `create_application_tracers', is used application:set_env(opentelemetry, register_loaded_applications, false), @@ -177,6 +182,11 @@ end_per_testcase(propagator_configuration_with_os_env, _Config) -> os:unsetenv("OTEL_PROPAGATORS"), _ = application:stop(opentelemetry), ok; +end_per_testcase(always_off, _Config) -> + application:unset_env(opentelemetry, sampler), + _ = application:stop(opentelemetry), + _ = application:unload(opentelemetry), + ok; end_per_testcase(_, _Config) -> application:unset_env(opentelemetry, attribute_value_length_limit), application:unset_env(opentelemetry, attribute_limits), @@ -356,6 +366,21 @@ new_propagator_configuration(_Config) -> [{otel_propagator_b3,b3multi},otel_propagator_baggage, otel_propagator_trace_context]}, opentelemetry:get_text_map_injector()), ok. +always_off(Config) -> + Tid = ?config(tid, Config), + SpanCtx1 = ?start_span(<<"span-1">>), + + ?set_current_span(SpanCtx1), + + otel_span:end_span(SpanCtx1), + + otel_tracer_provider:force_flush(), + + assert_not_exported(Tid, SpanCtx1), + + ok. + + force_flush(Config) -> Tid = ?config(tid, Config), SpanCtx1 = ?start_span(<<"span-1">>), @@ -894,7 +919,7 @@ non_recording_ets_table(_Config) -> SpanCtx1 = otel_tracer:start_span(Tracer, <<"span-1">>, #{}), ?assertMatch(true, SpanCtx1#span_ctx.is_recording), - AlwaysOff = otel_sampler:new(always_off), + AlwaysOff = otel_sampler:new(always_off, undefined), Tracer1 = {TracerModule, TracerConfig#tracer{sampler=AlwaysOff}}, SpanCtx2 = otel_tracer:start_span(Tracer1, <<"span-2">>, #{}), ?assertMatch(false, SpanCtx2#span_ctx.is_recording), diff --git a/apps/opentelemetry/test/otel_configuration_SUITE.erl b/apps/opentelemetry/test/otel_configuration_SUITE.erl index 6c1e77fd..afc80257 100644 --- a/apps/opentelemetry/test/otel_configuration_SUITE.erl +++ b/apps/opentelemetry/test/otel_configuration_SUITE.erl @@ -16,7 +16,7 @@ end, X)). all() -> - [empty_os_environment, sampler, sampler_parent_based, sampler_parent_based_zero, + [empty_os_environment, no_sampler, sampler, sampler_parent_based, sampler_parent_based_zero, sampler_trace_id, sampler_trace_id_default, sampler_parent_based_one, log_level, propagators, propagators_b3, propagators_b3multi, otlp_exporter, jaeger_exporter, zipkin_exporter, none_exporter, app_env_exporter, @@ -49,6 +49,12 @@ init_per_testcase(propagators_b3multi, Config) -> setup_env(Vars), + [{os_vars, Vars} | Config]; +init_per_testcase(no_sampler, Config) -> + Vars = [], + + setup_env(Vars), + [{os_vars, Vars} | Config]; init_per_testcase(sampler, Config) -> Vars = [{"OTEL_TRACES_SAMPLER", "parentbased_always_off"}], @@ -206,39 +212,45 @@ empty_os_environment(_Config) -> ok. +no_sampler(_Config) -> + ?assertMatch(error, + maps:find(sampler, maps:get(tracer_provider, otel_configuration:merge_with_os([])))), + + ok. + sampler(_Config) -> - ?assertMatch({parent_based, #{root := always_off}}, - maps:get(sampler, otel_configuration:merge_with_os([]))), + ?assertMatch({parent_based, #{root := {always_off, _}}}, + maps:get(sampler, maps:get(tracer_provider, otel_configuration:merge_with_os([])))), ok. sampler_parent_based(_Config) -> - ?assertMatch({parent_based, #{root := {trace_id_ratio_based, 0.5}}}, - maps:get(sampler, otel_configuration:merge_with_os([]))), + ?assertMatch({parent_based, #{root := {trace_id_ratio_based, #{ratio := 0.5}}}}, + maps:get(sampler, maps:get(tracer_provider, otel_configuration:merge_with_os([])))), ok. sampler_trace_id(_Config) -> - ?assertMatch({trace_id_ratio_based, 0.5}, - maps:get(sampler, otel_configuration:merge_with_os([]))), + ?assertMatch({trace_id_ratio_based, #{ratio := 0.5}}, + maps:get(sampler, maps:get(tracer_provider, otel_configuration:merge_with_os([])))), ok. sampler_trace_id_default(_Config) -> - ?assertMatch({trace_id_ratio_based, 1.0}, - maps:get(sampler, otel_configuration:merge_with_os([]))), + ?assertMatch({trace_id_ratio_based, #{ratio := 1.0}}, + maps:get(sampler, maps:get(tracer_provider, otel_configuration:merge_with_os([])))), ok. sampler_parent_based_one(_Config) -> - ?assertMatch({parent_based, #{root := {trace_id_ratio_based, 1.0}}}, - maps:get(sampler, otel_configuration:merge_with_os([]))), + ?assertMatch({parent_based, #{root := {trace_id_ratio_based, #{ratio := 1.0}}}}, + maps:get(sampler, maps:get(tracer_provider, otel_configuration:merge_with_os([])))), ok. sampler_parent_based_zero(_Config) -> - ?assertMatch({parent_based, #{root := {trace_id_ratio_based, +0.0}}}, - maps:get(sampler, otel_configuration:merge_with_os([]))), + ?assertMatch({parent_based, #{root := {trace_id_ratio_based, #{ratio := +0.0}}}}, + maps:get(sampler, maps:get(tracer_provider, otel_configuration:merge_with_os([])))), ok. From 0ff19d5f1dac0313754ff0bd2719c6f71e614a55 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Sun, 13 Jul 2025 11:53:14 -0400 Subject: [PATCH 10/15] cleanup eqwalizer and dialyzer warnings --- apps/opentelemetry/src/otel_configuration.erl | 37 ++++++++++++++----- apps/opentelemetry/src/otel_sampler.erl | 2 +- apps/opentelemetry/src/otel_span_limits.erl | 15 ++++---- 3 files changed, 37 insertions(+), 17 deletions(-) diff --git a/apps/opentelemetry/src/otel_configuration.erl b/apps/opentelemetry/src/otel_configuration.erl index 74540f49..01e8d68e 100644 --- a/apps/opentelemetry/src/otel_configuration.erl +++ b/apps/opentelemetry/src/otel_configuration.erl @@ -23,6 +23,7 @@ merge_with_os/1, merge_list_with_environment/3, transform/2, + defined_or_default/3, report_cb/1]). -type log_level() :: atom(). @@ -88,7 +89,13 @@ root => sampler()}} | {atom(), otel_config_properties:t()}. --type t() :: #{disabled => boolean(), +-type t() :: #{ + register_loaded_applications => boolean() | undefined, + create_application_tracers => boolean() | undefined, + id_generator => module(), + deny_list => list(), + + disabled => boolean(), log_level => log_level(), resource => #{attributes => opentelemetry:attributes_map()}, attribute_limits => attribute_limits(), @@ -99,6 +106,8 @@ sampler => sampler()} }. +-type old_t() :: map(). + %% structure before the standard otel declarative configuration %% -type old_t() :: #{sdk_disabled := boolean(), %% log_level := atom(), @@ -188,6 +197,16 @@ new() -> %% attribute_per_event_limit => 128, %% attribute_per_link_limit => 128}, old_t()). +defined_or_default(Key, Map, Default) -> + case maps:find(Key, Map) of + error -> + Default; + {ok, undefined} -> + Default; + {ok, Value} -> + Value + end. + %% constructs a config based on the flat format and merges with the new %% nested configuration from the declarative configuration SIG -spec merge_with_os(list()) -> t(). @@ -202,7 +221,7 @@ merge_with_os(Config) -> convert_to_new(OldConfig, Config). --spec convert_to_new(#{}, map()) -> t(). +-spec convert_to_new(map(), list()) -> t(). convert_to_new(OldConfig, Config) -> convert_tracer_provider( convert_propagator( @@ -226,7 +245,7 @@ convert_tracer_provider(OldConfig, Config) -> error -> {TracerProvider, OldConfig1} = convert_sampler(OldConfig), TracerProvider1 = convert_span_limits(TracerProvider, OldConfig1), - OldConfig1#{tracer_provider => maps:merge(TracerProvider1)}; + OldConfig1#{tracer_provider => TracerProvider1}; {Processors, OldConfig1} -> NewProcessors = lists:map(fun({otel_batch_processor, BatchConfig}) -> {batch, convert_batch(BatchConfig)}; @@ -420,11 +439,11 @@ convert_attribute_limits(OldConfig, Config) -> %% backwards compatability --spec span_limits(list(), t()) -> t(). +-spec span_limits(list(), old_t()) -> old_t(). span_limits(AppEnv, ConfigMap) -> merge_list_with_environment(config_mappings(span_limits), AppEnv, ConfigMap). --spec general(list(), t()) -> t(). +-spec general(list(), old_t()) -> old_t(). general(AppEnv, ConfigMap) -> Config = merge_list_with_environment(config_mappings(general_sdk), AppEnv, ConfigMap), @@ -445,9 +464,9 @@ general(AppEnv, ConfigMap) -> Bool end, undefined, Config), - ?assert_type(Config1, t()). + ?assert_type(Config1, old_t()). --spec sweeper(list(), t()) -> t(). +-spec sweeper(list(), old_t()) -> old_t(). sweeper(AppEnv, ConfigMap) -> DefaultSweeperConfig = maps:get(sweeper, ConfigMap, #{}), AppEnvSweeper = proplists:get_value(sweeper, AppEnv, #{}), @@ -458,7 +477,7 @@ sweeper(AppEnv, ConfigMap) -> DefaultSweeperConfig), ConfigMap#{sweeper => SweeperConfig}. --spec processors(list(), t()) -> t(). +-spec processors(list(), old_t()) -> old_t(). processors(AppEnv, ConfigMap) -> SpanProcessors = case transform(span_processor, proplists:get_value(span_processor, AppEnv)) of undefined -> @@ -525,7 +544,7 @@ is_default(Key, AppEnv, Mappings) -> %% sampler configuration is unique since it has the _ARG that is a sort of %% sub-configuration of the sampler config, and isn't a list. --spec sampler(list(), t()) -> t(). +-spec sampler(list(), old_t()) -> old_t(). sampler(AppEnv, ConfigMap) -> OSVar = "OTEL_TRACES_SAMPLER", Key = sampler, diff --git a/apps/opentelemetry/src/otel_sampler.erl b/apps/opentelemetry/src/otel_sampler.erl index 8914ff08..ef71cd25 100644 --- a/apps/opentelemetry/src/otel_sampler.erl +++ b/apps/opentelemetry/src/otel_sampler.erl @@ -100,7 +100,7 @@ %% @doc Returns a sampler based on the given specification. This is left for backwards %% compatibility. The function `new/2' should be used instead. --spec new(SamplerSpec :: sampler_spec()) -> t(). +-spec new(SamplerSpec :: sampler_spec() | always_on | always_off) -> t(). new(always_on) -> new({otel_sampler_always_on, #{}}); new(always_off) -> diff --git a/apps/opentelemetry/src/otel_span_limits.erl b/apps/opentelemetry/src/otel_span_limits.erl index 7b5cf718..6c90fd99 100644 --- a/apps/opentelemetry/src/otel_span_limits.erl +++ b/apps/opentelemetry/src/otel_span_limits.erl @@ -37,14 +37,14 @@ get() -> -spec set(otel_configuration:t()) -> ok. set(Config) -> - AttributeLimits = maps:get(attribute_limits, Config, #{}), - AttributeCountLimit = maps:get(attribute_count_limit, AttributeLimits, 128), - AttributeValueLengthLimit = maps:get(attribute_value_length_limit, AttributeLimits, infinity), + AttributeLimits = otel_configuration:defined_or_default(attribute_limits, Config, #{}), + AttributeCountLimit = otel_configuration:defined_or_default(attribute_count_limit, AttributeLimits, 128), + AttributeValueLengthLimit = otel_configuration:defined_or_default(attribute_value_length_limit, AttributeLimits, infinity), - EventCountLimit = maps:get(event_count_limit, Config, 128), - LinkCountLimit = maps:get(link_count_limit, Config, 128), - AttributePerEventLimit = maps:get(attribute_per_event_limit, Config, 128), - AttributePerLinkLimit = maps:get(attribute_per_link_limit, Config, 128), + EventCountLimit = otel_configuration:defined_or_default(event_count_limit, Config, 128), + LinkCountLimit = otel_configuration:defined_or_default(link_count_limit, Config, 128), + AttributePerEventLimit = otel_configuration:defined_or_default(attribute_per_event_limit, Config, 128), + AttributePerLinkLimit = otel_configuration:defined_or_default(attribute_per_link_limit, Config, 128), SpanLimits = #span_limits{attribute_count_limit=AttributeCountLimit, attribute_value_length_limit=AttributeValueLengthLimit, event_count_limit=EventCountLimit, @@ -53,6 +53,7 @@ set(Config) -> attribute_per_link_limit=AttributePerLinkLimit}, persistent_term:put(?SPAN_LIMITS_KEY, SpanLimits). + attribute_count_limit() -> get_limit(attribute_count_limit, ?MODULE:get()). From 03596b9d11bc503bc211bca0b4f664695b4f4425 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Fri, 18 Jul 2025 10:56:00 -0400 Subject: [PATCH 11/15] support OTEL_EXPERIMENTAL_CONFIG_FILE for passing in config file --- apps/opentelemetry/src/opentelemetry_app.erl | 14 ++++++++++++-- apps/opentelemetry/src/otel_tracer_server.erl | 2 +- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/apps/opentelemetry/src/opentelemetry_app.erl b/apps/opentelemetry/src/opentelemetry_app.erl index 4a3de242..24834225 100644 --- a/apps/opentelemetry/src/opentelemetry_app.erl +++ b/apps/opentelemetry/src/opentelemetry_app.erl @@ -24,8 +24,18 @@ -include_lib("opentelemetry_api/include/opentelemetry.hrl"). start(_StartType, _StartArgs) -> - Config = otel_configuration:merge_with_os( - application:get_all_env(opentelemetry)), + Env = application:get_all_env(opentelemetry), + Config = case os:getenv("OTEL_EXPERIMENTAL_CONFIG_FILE") of + false -> + case proplists:get_value(config_file, Env) of + undefined -> + otel_configuration:merge_with_os(Env); + File -> + otel_file_configuration:parse_file(File) + end; + File -> + otel_file_configuration:parse_file(File) + end, %% set the global propagators for HTTP based on the application env %% these get set even if the SDK is disabled diff --git a/apps/opentelemetry/src/otel_tracer_server.erl b/apps/opentelemetry/src/otel_tracer_server.erl index 4bfe2863..fe4e51ae 100644 --- a/apps/opentelemetry/src/otel_tracer_server.erl +++ b/apps/opentelemetry/src/otel_tracer_server.erl @@ -62,7 +62,7 @@ init([Name, SpanProcessorSup, Resource, Config]) -> TracerProviderConfig = maps:get(tracer_provider, Config, #{}), IdGeneratorModule = maps:get(id_generator, TracerProviderConfig, otel_id_generator), SamplerSpec = maps:get(sampler, TracerProviderConfig, {parent_based, #{root => {always_on, #{}}}}), - ct:pal("Sampler ~p", [SamplerSpec]), + DenyList = maps:get(deny_list, Config, []), Processors = maps:get(processors, TracerProviderConfig, [{otel_batch_processor, #{}}]), From 31cb4fdb736cb599814fb3da07a390500179493a Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Tue, 22 Jul 2025 06:46:53 -0400 Subject: [PATCH 12/15] merge exporter config during configuration resolution --- apps/opentelemetry/src/opentelemetry_app.erl | 4 +- apps/opentelemetry/src/otel_configuration.erl | 242 ++++++------------ .../test/otel_configuration_SUITE.erl | 4 +- .../src/otel_exporter_traces_otlp.erl | 3 +- .../test/opentelemetry_exporter_SUITE.erl | 16 +- 5 files changed, 95 insertions(+), 174 deletions(-) diff --git a/apps/opentelemetry/src/opentelemetry_app.erl b/apps/opentelemetry/src/opentelemetry_app.erl index 24834225..be615eb5 100644 --- a/apps/opentelemetry/src/opentelemetry_app.erl +++ b/apps/opentelemetry/src/opentelemetry_app.erl @@ -31,10 +31,10 @@ start(_StartType, _StartArgs) -> undefined -> otel_configuration:merge_with_os(Env); File -> - otel_file_configuration:parse_file(File) + otel_configuration:merge_with_env(otel_file_configuration:parse_file(File), Env) end; File -> - otel_file_configuration:parse_file(File) + otel_configuration:merge_with_env(otel_file_configuration:parse_file(File), Env) end, %% set the global propagators for HTTP based on the application env diff --git a/apps/opentelemetry/src/otel_configuration.erl b/apps/opentelemetry/src/otel_configuration.erl index 01e8d68e..5572c251 100644 --- a/apps/opentelemetry/src/otel_configuration.erl +++ b/apps/opentelemetry/src/otel_configuration.erl @@ -20,6 +20,7 @@ -module(otel_configuration). -export([new/0, + merge_with_env/2, merge_with_os/1, merge_list_with_environment/3, transform/2, @@ -32,6 +33,8 @@ attribute_count_limit => integer() | undefined }. +-type propagator() :: tracecontext | baggage | b3 | b3multi | jaeger | ottrace | atom(). + -type exporter_args() :: map(). -type exporter() :: {module(), exporter_args()}. @@ -51,33 +54,6 @@ otel_config_properties:t(). -type span_processors() :: #{span_processor_type() => span_processor()}. --type propagator() :: tracecontext | baggage | b3 | b3multi | jaeger | ottrace | atom(). - --type limits() :: #{%% Configure max attribute value size. Overrides .attribute_limits.attribute_value_length_limit. - %% Value must be non-negative. - %% If omitted or undefined, there is no limit. - attribute_value_length_limit => integer() | undefined, - %% Configure max attribute count. Overrides .attribute_limits.attribute_count_limit. - %% Value must be non-negative. - %% If omitted or undefined, 128 is used. - attribute_count_limit => integer() | undefined, - %% Configure max span event count. - %% Value must be non-negative. - %% If omitted or undefined, 128 is used. - event_count_limit => integer() | undefined, - %% Configure max span link count. - %% Value must be non-negative. - %% If omitted or undefined, 128 is used. - link_count_limit => integer() | undefined, - %% Configure max attributes per span event. - %% Value must be non-negative. - %% If omitted or undefined, 128 is used. - event_attribute_count_limit => integer() | undefined, - %% Configure max attributes per span link. - %% Value must be non-negative. - %% If omitted or undefined, 128 is used. - link_attribute_count_limit => integer() | undefined - }. -type sampler() :: {always_on, #{}} | {always_off, #{}} @@ -89,70 +65,39 @@ root => sampler()}} | {atom(), otel_config_properties:t()}. --type t() :: #{ +-type tracer_provider() :: #{processors => [span_processors()], + limits => otel_model_limits:t(), + sampler => sampler()}. + +-type resource() :: #{attributes => [#{name := unicode:unicode_binary(), + value := unicode:unicode_binary()}]}. + +-type propagators() :: #{composite => [propagator()], + composite_list => string()}. + +-type t() :: #{%% Erlang implementation specific configuration register_loaded_applications => boolean() | undefined, create_application_tracers => boolean() | undefined, id_generator => module(), deny_list => list(), - + sweeper := #{interval => integer() | infinity, + strategy => atom() | fun(), + span_ttl => integer() | infinity, + storage_size => integer() | infinity}, + resource_detectors => [atom()], + resource_detector_timeout => integer(), + + %% configuration based on otel declarative file configuration spec disabled => boolean(), log_level => log_level(), - resource => #{attributes => opentelemetry:attributes_map()}, + resource => resource(), attribute_limits => attribute_limits(), - propagator := #{composite => [propagator()], - composite_list => string()}, - tracer_provider => #{processors => [span_processors()], - limits => limits(), - sampler => sampler()} + propagator := propagators(), + tracer_provider => tracer_provider() }. -type old_t() :: map(). -%% structure before the standard otel declarative configuration -%% -type old_t() :: #{sdk_disabled := boolean(), -%% log_level := atom(), -%% register_loaded_applications := boolean() | undefined, -%% create_application_tracers := boolean() | undefined, -%% id_generator := module(), -%% deny_list := [atom()], - -%% resource_detectors := [module()], -%% resource_detector_timeout := integer(), - -%% attribute_count_limit := integer(), -%% attribute_value_length_limit := integer() | infinity, - -%% event_count_limit := integer(), -%% link_count_limit := integer(), - -%% attribute_per_event_limit := integer(), -%% attribute_per_link_limit := integer(), - -%% %% tracer provider -%% bsp_scheduled_delay_ms := integer() | undefined, -%% bsp_exporting_timeout_ms := integer() | undefined, -%% bsp_max_queue_size := integer() | undefined, -%% ssp_exporting_timeout_ms := integer() | undefined, -%% traces_exporter := {atom(), term()} | none | undefined, -%% processors := list(), -%% sampler := {atom(), term()}, - -%% text_map_propagators := [atom()], - -%% metrics_exporter := {atom(), term()} | none | undefined, -%% views := list(), %% TODO: type should be `[otel_meter_server:view_config]' -%% %% when Metrics are moved out of the experimental app -%% readers := [#{id := atom(), module => module(), config => map()}], -%% exemplars_enabled := boolean(), -%% exemplar_filter := always_on | always_off | trace_based, -%% metric_producers := [{module(), term()}], - -%% sweeper := #{interval => integer() | infinity, -%% strategy => atom() | fun(), -%% span_ttl => integer() | infinity, -%% storage_size => integer() | infinity} -%% }. - -export_type([t/0]). -include_lib("kernel/include/logger.hrl"). @@ -162,41 +107,6 @@ new() -> ?assert_type(#{}, t()). -%% old() -> -%% ?assert_type(#{sdk_disabled => false, -%% log_level => info, -%% register_loaded_applications => undefined, -%% create_application_tracers => undefined, -%% id_generator => otel_id_generator, -%% deny_list => [], -%% resource_detectors => [otel_resource_env_var, -%% otel_resource_app_env], -%% resource_detector_timeout => 5000, -%% bsp_scheduled_delay_ms => undefined, -%% bsp_exporting_timeout_ms => undefined, -%% bsp_max_queue_size => undefined, -%% ssp_exporting_timeout_ms => undefined, -%% text_map_propagators => [trace_context, baggage], -%% traces_exporter => {opentelemetry_exporter, #{}}, -%% metrics_exporter => {opentelemetry_exporter, #{}}, -%% views => [], -%% readers => [], -%% exemplars_enabled => false, -%% exemplar_filter => trace_based, -%% metric_producers => [], -%% processors => [{otel_batch_processor, ?BATCH_PROCESSOR_DEFAULTS}], -%% sampler => {parent_based, #{root => always_on}}, -%% sweeper => #{interval => timer:minutes(10), -%% strategy => drop, -%% span_ttl => timer:minutes(30), -%% storage_size => infinity}, -%% attribute_count_limit => 128, -%% attribute_value_length_limit => infinity, -%% event_count_limit => 128, -%% link_count_limit => 128, -%% attribute_per_event_limit => 128, -%% attribute_per_link_limit => 128}, old_t()). - defined_or_default(Key, Map, Default) -> case maps:find(Key, Map) of error -> @@ -207,13 +117,28 @@ defined_or_default(Key, Map, Default) -> Value end. +%% base configuration from either a json file or the app environment only +%% contains configuration from the otel config spec or for instrumentation +%% libraries. here we merge with our custom config values for things like +%% the sweeper +-spec merge_with_env(t(), [{atom(), term()}]) -> t(). +merge_with_env(Config, Env) -> + CustomConfig = lists:foldl(fun(F, Acc) -> + F(Env, Acc) + end, #{}, [fun custom/2, + fun sweeper/2]), + + maps:merge(Config, CustomConfig). + + %% constructs a config based on the flat format and merges with the new %% nested configuration from the declarative configuration SIG -spec merge_with_os(list()) -> t(). merge_with_os(Config) -> OldConfig = lists:foldl(fun(F, Acc) -> F(Config, Acc) - end, #{}, [fun span_limits/2, + end, #{}, [fun custom/2, + fun span_limits/2, fun general/2, fun sampler/2, fun processors/2, @@ -230,13 +155,6 @@ convert_to_new(OldConfig, Config) -> convert_disabled(OldConfig, Config), Config), Config), Config), Config). convert_tracer_provider(OldConfig, Config) -> - %% bsp_scheduled_delay_ms := integer() | undefined, - %% bsp_exporting_timeout_ms := integer() | undefined, - %% bsp_max_queue_size := integer() | undefined, - %% ssp_exporting_timeout_ms := integer() | undefined, - %% traces_exporter := {atom(), term()} | none | undefined, - %% processors := list(), - %% sampler := {atom(), term()}, case lists:keyfind(tracer_provider, 1, Config) of {tracer_provider, TracerProvider} -> OldConfig#{tracer_provider => TracerProvider}; @@ -405,47 +323,13 @@ convert_attribute_limits(OldConfig, Config) -> end end. - %% ConfigMap = new(), - - %% ConfigMap1 = lists:foldl(fun(F, Acc) -> - %% F(AppEnv, Acc) - %% end, ConfigMap, [fun resource/2, - %% fun attribute_limits/2, - %% fun propagator/2, - %% fun tracer_provider/2, - %% fun sweeper/2]), - - - -%% -spec resource(list(), t()) -> t(). -%% resource(AppEnv, ConfigMap) -> -%% Resource = proplists:get_value(resource, AppEnv, []), -%% ConfigMap. - -%% -spec attribute_limits(list(), t()) -> t(). -%% attribute_limits(AppEnv, ConfigMap) -> -%% AttributeLimits = proplists:get_value(attribute_limits, AppEnv, []), -%% ConfigMap. - -%% -spec propagator(list(), t()) -> t(). -%% propagator(AppEnv, ConfigMap) -> -%% Propagator = proplists:get_value(propagator, AppEnv, []), -%% ConfigMap. - -%% -spec tracer_provider(list(), t()) -> t(). -%% tracer_provider(AppEnv, ConfigMap) -> -%% TracerProvider = proplists:get_value(tracer_provider, AppEnv, []), -%% ConfigMap. - -%% backwards compatability - -spec span_limits(list(), old_t()) -> old_t(). span_limits(AppEnv, ConfigMap) -> merge_list_with_environment(config_mappings(span_limits), AppEnv, ConfigMap). --spec general(list(), old_t()) -> old_t(). -general(AppEnv, ConfigMap) -> - Config = merge_list_with_environment(config_mappings(general_sdk), AppEnv, ConfigMap), +-spec custom(list(), old_t()) -> old_t(). +custom(AppEnv, ConfigMap) -> + Config = merge_list_with_environment(config_mappings(custom), AppEnv, ConfigMap), %% merge the old `register_loaded_applications' with the new config key %% `create_application_tracers' that has replaced it @@ -466,6 +350,10 @@ general(AppEnv, ConfigMap) -> ?assert_type(Config1, old_t()). +-spec general(list(), old_t()) -> old_t(). +general(AppEnv, ConfigMap) -> + merge_list_with_environment(config_mappings(general_sdk), AppEnv, ConfigMap). + -spec sweeper(list(), old_t()) -> old_t(). sweeper(AppEnv, ConfigMap) -> DefaultSweeperConfig = maps:get(sweeper, ConfigMap, #{}), @@ -620,6 +508,15 @@ report_cb(#{source := transform, {"Transforming configuration value failed: os_var=~ts key=~ts transform=~ts value=~ts exception=~ts", [OSVar, Key, Transform, Value, otel_utils:format_exception(Kind, Reason, StackTrace)]}. +config_mappings(custom) -> + [%% `register_loaded_applications' is kept for backwards compatibility + {"OTEL_REGISTER_LOADED_APPLICATIONS", register_loaded_applications, boolean}, + {"OTEL_CREATE_APPLICATION_TRACERS", create_application_tracers, boolean}, + {"OTEL_ID_GENERATOR", id_generator, existing_atom}, + {"OTEL_DENY_LIST", deny_list, existing_atom_list}, + {"OTEL_RESOURCE_DETECTORS", resource_detectors, existing_atom_list}, + {"OTEL_RESOURCE_DETECTOR_TIMEOUT", resource_detector_timeout, integer} + ]; config_mappings(general_sdk) -> [{"OTEL_SDK_DISABLED", sdk_disabled, boolean}, {"OTEL_LOG_LEVEL", log_level, existing_atom}, @@ -682,7 +579,11 @@ transform(existing_atom_list, String) when is_list(String) -> end end, List); transform(exporter, Exporter) when Exporter =:= "otlp" ; Exporter =:= otlp -> - {opentelemetry_exporter, #{}}; + Config = merge_with_exporter_environment(#{}), + {opentelemetry_exporter, Config}; +transform(exporter, {opentelemetry_exporter, Config}) -> + Config1 = merge_with_exporter_environment(Config), + {opentelemetry_exporter, Config1}; transform(exporter, Exporter) when Exporter =:= "jaeger" ; Exporter =:= jaeger -> ?LOG_WARNING("configuring jaeger exporter through OTEL_TRACES_EXPORTER is not yet supported ", []), none; @@ -822,6 +723,11 @@ transform(span_processor, batch) -> {otel_batch_processor, #{}}; transform(span_processor, simple) -> {otel_simple_processor, #{}}; + +%% merge old processor config of OTLP exporter with opentelemetry_exporter app env and OS vars +transform(span_processor, {Module, Opts=#{exporter := {ExporterModule=opentelemetry_exporter, ExporterOpts}}}) -> + ExporterOpts1 = otel_exporter_traces_otlp:merge_with_environment(ExporterOpts), + {Module, Opts#{exporter => {ExporterModule, ExporterOpts1}}}; transform(span_processor, SpanProcessor) -> SpanProcessor; transform(readers, Readers) -> @@ -853,9 +759,21 @@ probability_string_to_float(Probability) -> catch error:badarg when Probability =:= "1" -> 1.0; - error:badarg when Probability =:= "0" -> + error:badarg when Probability =:= "0" -> 0.0; error:badarg -> ?LOG_WARNING("Unable to convert $OTEL_TRACE_SAMPLER_ARG string value ~ts to float, using 1.0"), 1.0 end. + +%% merge the otlp exporter configs with those found in the exporter app itself +%% this is deprecated as the new configuration model defines all config in the SDK +merge_with_exporter_environment(Config) -> + case application:load(opentelemetry_exporter) of + ok -> + otel_exporter_traces_otlp:merge_with_environment(Config); + {error, _} -> + ?LOG_WARNING("Unable to load opentelemetry_exporter application but configured to " + "use otlp exporter. Check your dependencies or configuration."), + Config + end. diff --git a/apps/opentelemetry/test/otel_configuration_SUITE.erl b/apps/opentelemetry/test/otel_configuration_SUITE.erl index afc80257..5d62cbbb 100644 --- a/apps/opentelemetry/test/otel_configuration_SUITE.erl +++ b/apps/opentelemetry/test/otel_configuration_SUITE.erl @@ -409,7 +409,9 @@ span_processors(_Config) -> ?assertMatch(#{tracer_provider := #{processors := [{batch, #{exporter := {opentelemetry_exporter, - #{endpoints := ["https://example.com"]}}, + #{endpoints := [#{scheme := "https", + path := "/v1/traces", + host := "example.com"}]}}, export_timeout := 2, max_queue_size := 1, schedule_delay := 15000}}]}}, diff --git a/apps/opentelemetry_exporter/src/otel_exporter_traces_otlp.erl b/apps/opentelemetry_exporter/src/otel_exporter_traces_otlp.erl index 40d39775..1837b750 100644 --- a/apps/opentelemetry_exporter/src/otel_exporter_traces_otlp.erl +++ b/apps/opentelemetry_exporter/src/otel_exporter_traces_otlp.erl @@ -119,8 +119,7 @@ %% @doc Initialize the exporter based on the provided configuration. -spec init(otel_exporter_otlp:opts()) -> {ok, #state{}}. init(Opts) -> - Opts1 = merge_with_environment(Opts), - case otel_exporter_otlp:init(Opts1) of + case otel_exporter_otlp:init(Opts) of {ok, #{channel := Channel, channel_pid := ChannelPid, endpoints := Endpoints, diff --git a/apps/opentelemetry_exporter/test/opentelemetry_exporter_SUITE.erl b/apps/opentelemetry_exporter/test/opentelemetry_exporter_SUITE.erl index 6ecd5cd0..16bf6973 100644 --- a/apps/opentelemetry_exporter/test/opentelemetry_exporter_SUITE.erl +++ b/apps/opentelemetry_exporter/test/opentelemetry_exporter_SUITE.erl @@ -373,14 +373,16 @@ verify_export(Config) -> http_protobuf -> 4318 end, - {ok, State} = opentelemetry_exporter:init(#{protocol => Protocol, - compression => Compression, - endpoints => [{http, "localhost", Port, []}]}), - + ExporterOpts = otel_exporter_traces_otlp:merge_with_environment(#{protocol => Protocol, + compression => Compression, + endpoints => [{http, "localhost", Port, []}]}), + {ok, State} = opentelemetry_exporter:init(ExporterOpts), + + ExporterOpts1 = otel_exporter_traces_otlp:merge_with_environment(#{protocol => Protocol, + compression => Compression, + endpoints => [{http, "localhost", Port, []}]}), %% regression test. adding the httpc profile meant that init'ing more than once would crash - {ok, _State} = opentelemetry_exporter:init(#{protocol => Protocol, - compression => Compression, - endpoints => [{http, "localhost", Port, []}]}), + {ok, _State} = opentelemetry_exporter:init(ExporterOpts1), Tid = ets:new(span_tab, [duplicate_bag, {keypos, #span.instrumentation_scope}]), From aa229512bb373e2add560e314693a48980578608 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Tue, 29 Jul 2025 07:05:43 -0400 Subject: [PATCH 13/15] configuration: add types for otlp_http and otlp_grpc --- apps/opentelemetry/src/otel_configuration.erl | 94 ++++++++++++++++++- apps/opentelemetry/src/otel_exporter.erl | 2 + .../src/otel_exporter_otlp.erl | 3 +- 3 files changed, 94 insertions(+), 5 deletions(-) diff --git a/apps/opentelemetry/src/otel_configuration.erl b/apps/opentelemetry/src/otel_configuration.erl index 5572c251..fbe40671 100644 --- a/apps/opentelemetry/src/otel_configuration.erl +++ b/apps/opentelemetry/src/otel_configuration.erl @@ -25,6 +25,7 @@ merge_list_with_environment/3, transform/2, defined_or_default/3, + convert_exporter/1, report_cb/1]). -type log_level() :: atom(). @@ -35,9 +36,38 @@ -type propagator() :: tracecontext | baggage | b3 | b3multi | jaeger | ottrace | atom(). +-type compression() :: gzip | none | undefined. + +-type otlp_http() :: #{ + endpoint => uri:uri_string(), + certificate_file => file:filename_all() | undefined, + client_key_file => file:filename_all() | undefined, + client_certificate_file => file:filename_all() | undefined, + compression => compression(), + timeout => integer(), + headers => #{unicode:latin1_binary() => unicode:latin1_binary()} | undefined, + headers_list => unicode:latin1_binary(), + encoding => protobuf | undefined + }. + +-type otlp_grpc() :: #{ + endpoint => uri:uri_string(), + certificate_file => file:filename_all() | undefined, + client_key_file => file:filename_all() | undefined, + client_certificate_file => file:filename_all() | undefined, + compression => compression(), + timeout => integer(), + headers => #{unicode:latin1_binary() => unicode:latin1_binary()} | undefined, + headers_list => unicode:latin1_binary(), + insecure => boolean() + }. + -type exporter_args() :: map(). --type exporter() :: {module(), exporter_args()}. +-type exporter() :: {otlp_http, otlp_http()} | + {otlp_grpc, otlp_grpc()} | + {module(), exporter_args()}. + -type batch_span_processor() :: #{schedule_delay := integer(), export_timeout := integer(), @@ -52,7 +82,7 @@ -type span_processor() :: batch_span_processor() | simple_span_processor() | otel_config_properties:t(). --type span_processors() :: #{span_processor_type() => span_processor()}. +-type span_processors() :: {span_processor_type(), span_processor()}. -type sampler() :: {always_on, #{}} @@ -241,8 +271,61 @@ update_processor_config(Config, Options) -> end end, Config, Options). -convert_exporter(ExporterConfig) -> - ExporterConfig. +%% in this case convert new configuration file structure to a `{module(), args()}' form +convert_exporter({otlp_grpc, GrpcConfig=#{endpoint := Endpoint}}) -> + Compression = maps:get(compression, GrpcConfig, undefined), + Timeout = maps:get(timeout, GrpcConfig, 3000), + SSLOptions = exporter_ssl_options(GrpcConfig), + Headers = exporter_headers(GrpcConfig), + {opentelemetry_exporter, #{endpoints => [Endpoint], + headers => Headers, + protocol => grpc, + ssl_options => SSLOptions, + compression => Compression, + timeout => Timeout}}; +convert_exporter({otlp_http, HttpConfig=#{endpoint := Endpoint}}) -> + Compression = maps:get(compression, HttpConfig, undefined), + Timeout = maps:get(timeout, HttpConfig, 3000), + SSLOptions = exporter_ssl_options(HttpConfig), + Headers = exporter_headers(HttpConfig), + {opentelemetry_exporter, #{endpoints => [Endpoint], + headers => Headers, + protocol => http, + ssl_options => SSLOptions, + compression => Compression, + timeout => Timeout}}. + +exporter_headers(Config) -> + Headers = maps:get(headers, Config, []), + HeadersList = maps:get(headers_list, Config, ""), + + HeadersFromList = transform(key_value_list, HeadersList), + + Headers ++ HeadersFromList. + + +%% technically this is only an option for grpc for some reason but we support it on +%% `otlp_http' too +exporter_ssl_options(#{insecure := true}) -> + []; +%% both `client_key_file' and `client_cert_file' must be set for ssl to be used +exporter_ssl_options(Config=#{client_key_file := ClientKeyFile, + client_certificate_file := ClientCertFile}) when + ClientKeyFile =/= undefined andalso + ClientCertFile =/= undefined -> + %% if `certificate_file' is missing then use OS certs + CACerts = case maps:find(certificate_file, Config) of + {ok, undefined} -> + {cacerts, public_key:cacerts_get()}; + error -> + {cacerts, public_key:cacerts_get()}; + {ok, File} -> + {cacertfile, File} + end, + + [{keyfile, ClientKeyFile}, {certfile, ClientCertFile}, CACerts]; +exporter_ssl_options(_) -> + []. %% convert the old `text_map_propagators' config into the new `propagator' config %% based on the declarative file configuration of otel @@ -293,6 +376,9 @@ convert_resource(OldConfig, Config) -> OldConfig end. +%% ignore `sdk_disabled' if `disabled' is set +convert_disabled(OldConfig, Config=#{disabled := _}) -> + OldConfig; convert_disabled(OldConfig, Config) -> case maps:take(sdk_disabled, OldConfig) of {Value, OldConfig1} -> diff --git a/apps/opentelemetry/src/otel_exporter.erl b/apps/opentelemetry/src/otel_exporter.erl index 90f6213f..a451713c 100644 --- a/apps/opentelemetry/src/otel_exporter.erl +++ b/apps/opentelemetry/src/otel_exporter.erl @@ -44,6 +44,8 @@ -include_lib("kernel/include/logger.hrl"). +init(Exporter={Type, _}) when Type =:= otlp_grpc orelse Type =:= otlp_http -> + init(otel_configuration:convert_exporter(Exporter)); init({ExporterModule, Config}) when is_atom(ExporterModule) -> try ExporterModule:init(Config) of {ok, ExporterState} when ExporterModule =:= opentelemetry_exporter -> diff --git a/apps/opentelemetry_exporter/src/otel_exporter_otlp.erl b/apps/opentelemetry_exporter/src/otel_exporter_otlp.erl index 55e4a491..8365663b 100644 --- a/apps/opentelemetry_exporter/src/otel_exporter_otlp.erl +++ b/apps/opentelemetry_exporter/src/otel_exporter_otlp.erl @@ -52,7 +52,8 @@ -type opts() :: #{endpoints => [endpoint()], headers => headers(), protocol => protocol(), - ssl_options => list()}. + ssl_options => list(), + insecure => boolean()}. -export_type([opts/0, headers/0, From d5b1284fca0e2fe15a815ea9607fef83a3f34ae0 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Thu, 7 Aug 2025 06:25:48 -0400 Subject: [PATCH 14/15] add file configuration modules for parsing json config --- .../src/otel_file_configuration.erl | 18 ++++++ .../test/otel_file_configuration_SUITE.erl | 55 +++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 apps/opentelemetry/src/otel_file_configuration.erl create mode 100644 apps/opentelemetry/test/otel_file_configuration_SUITE.erl diff --git a/apps/opentelemetry/src/otel_file_configuration.erl b/apps/opentelemetry/src/otel_file_configuration.erl new file mode 100644 index 00000000..656da5b4 --- /dev/null +++ b/apps/opentelemetry/src/otel_file_configuration.erl @@ -0,0 +1,18 @@ +-module(otel_file_configuration). + +-export([parse_file/1, + parse_binary/1]). + +%% +-spec parse_file(file:filename_all()) -> otel_configuration:t(). +parse_file(JsonFile) -> + {ok, File} = file:read_file(JsonFile), + parse_binary(File). + +%% TODO: Log a warning on failure to parse and return empty configuration +-spec parse_binary(binary()) -> otel_configuration:t(). +parse_binary(Bin) -> + Push = fun(Key, Value, Acc) -> [{binary_to_atom(Key), Value} | Acc] end, + {Json, ok, <<>>} = json:decode(Bin, ok, #{object_push => Push, null => undefined}), + + Json. diff --git a/apps/opentelemetry/test/otel_file_configuration_SUITE.erl b/apps/opentelemetry/test/otel_file_configuration_SUITE.erl new file mode 100644 index 00000000..b4836809 --- /dev/null +++ b/apps/opentelemetry/test/otel_file_configuration_SUITE.erl @@ -0,0 +1,55 @@ +-module(otel_file_configuration_SUITE). + +-compile(export_all). + +-include_lib("stdlib/include/assert.hrl"). +-include_lib("common_test/include/ct.hrl"). + +all() -> + [parse_test, app_env_test, os_env_test]. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +parse_test(Config) -> + DataDir = ?config(data_dir, Config), + JsonFile = filename:join(DataDir, "out.json"), + + Json = otel_file_configuration:parse_file(JsonFile), + + ?assertMatch(#{disabled := false}, Json). + +app_env_test(Config) -> + try + DataDir = ?config(data_dir, Config), + JsonFile = filename:join(DataDir, "out.json"), + + application:load(opentelemetry), + application:set_env(opentelemetry, config_file, JsonFile), + + {ok, _} = application:ensure_all_started(opentelemetry) + + after + application:unset_env(opentelemetry, config_file), + application:stop(opentelemetry) + end, + + ok. + +os_env_test(Config) -> + try + DataDir = ?config(data_dir, Config), + JsonFile = filename:join(DataDir, "out.json"), + + os:putenv("OTEL_EXPERIMENTAL_CONFIG_FILE", JsonFile), + + {ok, _} = application:ensure_all_started(opentelemetry) + after + os:unsetenv("OTEL_EXPERIMENTAL_CONFIG_FILE"), + application:stop(opentelemetry) + end, + + ok. From e65e85ff7d2d957cf31453ac725390cb553a5fdb Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Thu, 7 Aug 2025 06:30:33 -0400 Subject: [PATCH 15/15] update changelog for config file support --- CHANGELOG.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd12d504..725ce3ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 #### Added -- +- Support `OTEL_EXPERIMENTAL_CONFIG_FILE` environment variable to specify a json + file assumed to already be validated against the Otel file configuration + jsonschema. +- Support the new file configuration schema as Erlang application environment + configuration in addition to as json. #### Changes @@ -19,6 +23,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 structure resource attribute keys with dots. Meaning use a key of `service.name` instead of `#{service => #{name => ...}}` to represent `service.name` attribute in the resource. +- Options passed to the start of a tracer server and span processor have changed + to correspond to the file configuration schema. ## Experimental API 0.5.2 - 2024-11-22