Merge pull request #341 from tsloughter/protocol-compression-env

update and add exporter options

Author: tsloughter

CHANGELOG.md

@@ -7,6 +7,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### [Exporter]
+
+#### Added
+
+- New `opentelemetry_exporter` application environment options:
+  - `otlp_protocol`: The transport protocol, supported values: `grpc` and `http_protobuf`. Defaults to `http_protobuf`.
+  - `otlp_traces_protocol`: The transport protocol to use for exporting traces, supported values: `grpc` and `http_protobuf`. Defaults to `http_protobuf`.
+  - `otlp_compression`: Compression type to use, supported values: `gzip`. Defaults to no compression.
+  - `otlp_traces_compression`: Compression type to use for exporting traces, supported values: `gzip`. Defaults to no compression.
+
+- New environment variable options:
+  - `OTEL_EXPORTER_OTLP_PROTOCOL`: The transport protocol to use, supported values: `grpc` and `http_protobuf`. Defaults to `http_protobuf`
+  - `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL`: The transport protocol to use for exporting traces, supported values: `grpc` and `http_protobuf`. Defaults to `http_protobuf`.
+  - `OTEL_EXPORTER_OTLP_COMPRESSION`: Compression to use, supported value: gzip. Defaults to no compression.
+  - `OTEL_EXPORTER_OTLP_TRACES_COMPRESSION`: Compression to use when exporting traces, supported value: gzip. Defaults to no compression.
+
 ## [API 1.0.0-rc.4.1] - 2021-12-28
 
 ##### Fixed

apps/opentelemetry_exporter/README.md

@@ -51,19 +51,27 @@ config :opentelemetry, :processors,
 - `otlp_endpoint`: The URL to send traces and metrics to, for traces the path `v1/traces` is appended to the path in the URL.
 - `otlp_traces_endpoint`: URL to send only traces to. This takes precedence for exporting traces and the path of the URL is kept as is, no suffix is appended.
 - `otlp_headers`: List of additional headers (`[{unicode:chardata(), unicode:chardata()}]`) to add to export requests.
-- `otlp_traces_headers`: Additional headers (`[{unicode:chardata(), unicode:chardata()}]`) to add to only trace export requests.
-    
+- `otlp_traces_headers`: Additional headers (`[{unicode:chardata(),
+  unicode:chardata()}]`) to add to only trace export requests.
+- `otlp_protocol`: The transport protocol, supported values: `grpc` and `http_protobuf`. Defaults to `http_protobuf`.
+- `otlp_traces_protocol`: The transport protocol to use for exporting traces, supported values: `grpc` and `http_protobuf`. Defaults to `http_protobuf`.
+- `otlp_compression`: Compression type to use, supported values: `gzip`. Defaults to no compression.
+- `otlp_traces_compression`: Compression type to use for exporting traces, supported values: `gzip`. Defaults to no compression.
 
 ``` erlang
 {opentelemetry_exporter,
-  [{otlp_endpoint, "https://api.honeycomb.io:443"},
+  [{otlp_protocol, grpc},
+   {otlp_compression, gzip},
+   {otlp_endpoint, "https://api.honeycomb.io:443"},
    {otlp_headers, [{"x-honeycomb-dataset", "experiments"}]}]}
 ```
 
 An Elixir release uses `releases.exs`:
 
 ``` elixir
 config :opentelemetry_exporter,
+  otlp_protocol: :grpc,
+  otlp_compression: :gzip,
   otlp_endpoint: "https://api.honeycomb.io:443",
   otlp_headers: [{"x-honeycomb-dataset", "experiments"}]
 ```
@@ -87,11 +95,16 @@ for more information on securing HTTP requests in Erlang.
 - `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`: URL to send only traces to. This takes precedence for exporting traces and the path of the URL is kept as is, no suffix is appended.
 - `OTEL_EXPORTER_OTLP_HEADERS`: List of additional headers to add to export requests.
 - `OTEL_EXPORTER_OTLP_TRACES_HEADERS`: Additional headers to add to only trace export requests.
+- `OTEL_EXPORTER_OTLP_PROTOCOL`: The transport protocol to use, supported values: `grpc` and `http_protobuf`. Defaults to `http_protobuf`
+- `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL`: The transport protocol to use for exporting traces, supported values: `grpc` and `http_protobuf`. Defaults to `http_protobuf`.
+- `OTEL_EXPORTER_OTLP_COMPRESSION`: Compression to use, supported value: gzip. Defaults to no compression.
+- `OTEL_EXPORTER_OTLP_TRACES_COMPRESSION`: Compression to use when exporting traces, supported value: gzip. Defaults to no compression.
 
 Example usage of setting the environment variables:
 ```
 OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=https://api.honeycomb.io:443
-OTEL_EXPORTER_OTLP_PROTOCOL=grpc
+OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=grpc
+OTEL_EXPORTER_OTLP_TRACES_COMPRESSION=gzip
 OTEL_EXPORTER_OTLP_TRACES_HEADERS=x-honeycomb-team=<HONEYCOMB API TOKEN>,x-honeycomb-dataset=experiments
 ```
 

apps/opentelemetry_exporter/src/opentelemetry_exporter.erl

@@ -24,14 +24,29 @@
 %%     `otlp_endpoint': The URL to send traces and metrics to, for traces the
 %%     path `v1/traces' is appended to the path in the URL.
 %%   </li>
-%%   <li>`otlp_traces_endpoint': URL to send only traces to. This takes precedence
+%%   <li>
+%%     `otlp_traces_endpoint': URL to send only traces to. This takes precedence
 %%     for exporting traces and the path of the URL is kept as is, no suffix is
 %%     appended.
 %%   </li>
-%%   <li>`otlp_headers': List of additional headers (`[{unicode:chardata(), unicode:chardata()}]') to add to export requests.</li>
+%%   <li>
+%%     `otlp_headers': List of additional headers (`[{unicode:chardata(), unicode:chardata()}]') to add to export requests.
+%%   </li>
 %%   <li>
 %%     `otlp_traces_headers': Additional headers (`[{unicode:chardata(), unicode:chardata()}]') to add to only trace export requests.
 %%   </li>
+%%   <li>
+%%     `otlp_protocol': The transport protocol, supported values: `grpc' and `http_protobuf'. Defaults to `http_protobuf'.
+%%   </li>
+%%   <li>
+%%     `otlp_traces_protocol': The transport protocol to use for exporting traces, supported values: `grpc' and `http_protobuf'. Defaults to `http_protobuf'
+%%   </li>
+%%   <li>
+%%     `otlp_compression': Compression type to use, supported values: `gzip'. Defaults to no compression.
+%%   </li>
+%%   <li>
+%%     `otlp_traces_compression': Compression type to use for exporting traces, supported values: `gzip'. Defaults to no compression.
+%%   </li>
 %% </ul>
 %%
 %% There also corresponding OS environment variables can also set those
@@ -42,6 +57,10 @@
 %%   <li>`OTEL_EXPORTER_OTLP_TRACES_ENDPOINT': URL to send only traces to. This takes precedence for exporting traces and the path of the URL is kept as is, no suffix is appended.</li>
 %%   <li>`OTEL_EXPORTER_OTLP_HEADERS': List of additional headers to add to export requests.</li>
 %%   <li>`OTEL_EXPORTER_OTLP_TRACES_HEADERS': Additional headers to add to only trace export requests.</li>
+%%   <li>`OTEL_EXPORTER_OTLP_PROTOCOL': The transport protocol to use, supported values: `grpc' and `http_protobuf'. Defaults to `http_protobuf'.</li>
+%%   <li>`OTEL_EXPORTER_OTLP_TRACES_PROTOCOL': The transport protocol to use for exporting traces, supported values: `grpc' and `http_protobuf'. Defaults to `http_protobuf'.</li>
+%%   <li>`OTEL_EXPORTER_OTLP_COMPRESSION': Compression to use, supported value: gzip. Defaults to no compression./li>
+%%   <li>`OTEL_EXPORTER_OTLP_TRACES_COMPRESSION': Compression to use when exporting traces, supported value: gzip. Defaults to no compression.</li>
 %% </ul>
 %%
 %% @end
@@ -74,9 +93,9 @@
 
 -define(DEFAULT_ENDPOINTS, [#{host => "localhost",
                               path => [],
-                              port => 4317,
-                              scheme => "https"}]).
--define(DEFAULT_PORT, 4317).
+                              port => 4318,
+                              scheme => "http"}]).
+-define(DEFAULT_PORT, 4318).
 -define(DEFAULT_TRACES_PATH, "v1/traces").
 
 -type headers() :: [{unicode:chardata(), unicode:chardata()}].
@@ -310,7 +329,11 @@ merge_with_environment(Opts) ->
     Config = #{otlp_endpoint => undefined,
                otlp_traces_endpoint => undefined,
                otlp_headers => undefined,
-               otlp_traces_headers => undefined},
+               otlp_traces_headers => undefined,
+               otlp_protocol => undefined,
+               otlp_traces_protocol => undefined,
+               otlp_compression => undefined,
+               otlp_traces_compression => undefined},
 
     AppEnv = application:get_all_env(opentelemetry_exporter),
     AppOpts = otel_configuration:merge_list_with_environment(config_mapping(), AppEnv, Config),
@@ -320,7 +343,13 @@ merge_with_environment(Opts) ->
     Opts2 = update_opts(otlp_traces_endpoint, endpoints, ?DEFAULT_ENDPOINTS, AppOpts, Opts1, fun maybe_to_list/1),
 
     Opts3 = update_opts(otlp_headers, headers, [], AppOpts, Opts2),
-    update_opts(otlp_traces_headers, headers, [], AppOpts, Opts3).
+    Opts4 = update_opts(otlp_traces_headers, headers, [], AppOpts, Opts3),
+
+    Opts5 = update_opts(otlp_protocol, protocol, http_protobuf, AppOpts, Opts4),
+    Opts6 = update_opts(otlp_traces_protocol, protocol, http_protobuf, AppOpts, Opts5),
+
+    Opts7 = update_opts(otlp_compression, compression, undefined, AppOpts, Opts6),
+    update_opts(otlp_traces_compression, compression, undefined, AppOpts, Opts7).
 
 maybe_to_list(E) when is_list(E) ->
     case io_lib:printable_list(E) of
@@ -376,13 +405,24 @@ config_mapping() ->
 
      %% headers to include in requests the exporter makes over the Otel protocol
      {"OTEL_EXPORTER_OTLP_HEADERS", otlp_headers, key_value_list},
-     {"OTEL_EXPORTER_OTLP_TRACES_HEADERS", otlp_traces_headers, key_value_list}
+     {"OTEL_EXPORTER_OTLP_TRACES_HEADERS", otlp_traces_headers, key_value_list},
      %% {"OTEL_EXPORTER_OTLP_METRICS_HEADERS", otlp_metrics_headers, "", key_value_list}
 
-     %% the following are not yet defined in the spec
-     %% {"OTEL_EXPORTER_OTLP_PROTOCOL", exporter_otlp_protocol, "", string},
-     %% {"OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", exporter_otlp_traces_protocol, "", string},
-     %% {"OTEL_EXPORTER_OTLP_METRICS_PROTOCOL", exporter_otlp_metrics_protocol, "", string}
+     {"OTEL_EXPORTER_OTLP_PROTOCOL", otlp_protocol, existing_atom},
+     {"OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", otlp_traces_protocol, existing_atom},
+     %% {"OTEL_EXPORTER_OTLP_METRICS_PROTOCOL", exporter_otlp_metrics_protocol, string}
+
+     {"OTEL_EXPORTER_OTLP_COMPRESSION", otlp_compression, existing_atom},
+     {"OTEL_EXPORTER_OTLP_TRACES_COMPRESSION", otlp_traces_compression, existing_atom}
+     %% {"OTEL_EXPORTER_OTLP_METRICS_COMPRESSION", otlp_metrics_compression, existing_atom}
+
+     %% {"OTEL_EXPORTER_OTLP_CERTIFICATE", otlp_certificate, path},
+     %% {"OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE", otlp_traces_certificate, path},
+     %% {"OTEL_EXPORTER_OTLP_METRICS_CERTIFICATE", otlp_metrics_certificate, path},
+
+     %% {"OTEL_EXPORTER_OTLP_TIMEOUT", otlp_timeout, integer},
+     %% {"OTEL_EXPORTER_OTLP_TRACES_TIMEOUT", otlp_traces_timeout, integer},
+     %% {"OTEL_EXPORTER_OTLP_METRICS_TIMEOUT", otlp_metrics_timeout, integer}
     ].
 
 tab_to_proto(Tab, Resource) ->

apps/opentelemetry_exporter/test/opentelemetry_exporter_SUITE.erl

@@ -77,7 +77,7 @@ configuration(_Config) ->
         ?assertMatch([#{scheme := "https", host := "localhost", port := 443, path := "/used/path"}],
                      opentelemetry_exporter:endpoints(<<"https://localhost:443/used/path">>, [])),
 
-        ?assertMatch([#{scheme := "http", host := "localhost", port := 4317, path := []}],
+        ?assertMatch([#{scheme := "http", host := "localhost", port := 4318, path := []}],
                      opentelemetry_exporter:endpoints("http://localhost", [])),
 
         ?assertMatch([], opentelemetry_exporter:endpoints("://badendpoint", [])),
@@ -97,15 +97,19 @@ configuration(_Config) ->
         ?assertEqual(#{endpoints =>
                            [#{host => "localhost", path => "/v1/traces", port => 4343,
                               scheme => "http"}],
-                       headers => [{"key1", "value1"}]},
+                       headers => [{"key1", "value1"}],
+                       compression => undefined,
+                       protocol => http_protobuf},
                      opentelemetry_exporter:merge_with_environment(#{endpoints => []})),
 
         os:putenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4343/internal"),
         os:putenv("OTEL_EXPORTER_OTLP_HEADERS", "key1=value1"),
         ?assertEqual(#{endpoints =>
                            [#{host => "localhost", path => "/internal/v1/traces", port => 4343,
                               scheme => "http"}],
-                       headers => [{"key1", "value1"}]},
+                       headers => [{"key1", "value1"}],
+                       compression => undefined,
+                       protocol => http_protobuf},
                      opentelemetry_exporter:merge_with_environment(#{endpoints => []})),
 
         %% TRACES_ENDPOINT takes precedence
@@ -114,16 +118,22 @@ configuration(_Config) ->
         ?assertEqual(#{endpoints =>
                            [#{host => "localhost", path => "/traces/path", port => 5353,
                               scheme => "http"}],
-                       headers => [{"key2", "value2"}]},
+                       headers => [{"key2", "value2"}],
+                       compression => undefined,
+                       protocol => http_protobuf},
                      opentelemetry_exporter:merge_with_environment(#{endpoints => []})),
 
+        os:putenv("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc"),
+        ?assertMatch(#{protocol := grpc},
+                     opentelemetry_exporter:merge_with_environment(#{})),
 
         ok
     after
         os:unsetenv("OTEL_EXPORTER_OTLP_ENDPOINT"),
         os:unsetenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"),
         os:unsetenv("OTEL_EXPORTER_OTLP_HEADERS"),
-        os:unsetenv("OTEL_EXPORTER_OTLP_TRACES_HEADERS")
+        os:unsetenv("OTEL_EXPORTER_OTLP_TRACES_HEADERS"),
+        os:unsetenv("OTEL_EXPORTER_OTLP_PROTOCOL")
     end.
 
 ets_instrumentation_info(_Config) ->
@@ -228,11 +238,12 @@ verify_export(Config) ->
     os:putenv("OTEL_RESOURCE_ATTRIBUTES", "service.name=my-test-service,service.version=98da75ea6d38724743bf42b45565049238d86b3f"),
     Protocol = ?config(protocol, Config),
     Compression = ?config(compression, Config),
+
     Port = case Protocol of
                grpc ->
                    4317;
                http_protobuf ->
-                   55681
+                   4318
            end,
     {ok, State} = opentelemetry_exporter:init(#{protocol => Protocol,
                                                 compression => Compression,

config/otel-collector-config.yaml

@@ -5,7 +5,7 @@ receivers:
       grpc:
         endpoint: "0.0.0.0:4317"
       http:
-        endpoint: "0.0.0.0:55681"
+        endpoint: "0.0.0.0:4318"
 processors:
   batch:
     send_batch_size: 1024

docker-compose.yml

@@ -5,8 +5,8 @@ services:
     command: ["--config=/conf/otel-collector-config.yaml"]
     privileged: true
     ports:
-      - 55681:55681
       - 4317:4317
+      - 4318:4318
     volumes:
       - ./config/otel-collector-config.yaml:/conf/otel-collector-config.yaml
     links: