diff --git a/instrumentation/opentelemetry_finch/lib/opentelemetry_finch.ex b/instrumentation/opentelemetry_finch/lib/opentelemetry_finch.ex index ce9bbf67..81773905 100644 --- a/instrumentation/opentelemetry_finch/lib/opentelemetry_finch.ex +++ b/instrumentation/opentelemetry_finch/lib/opentelemetry_finch.ex @@ -13,13 +13,116 @@ defmodule OpentelemetryFinch do # ... end + ## Semantic Conventions + + Module follows [Client HTTP Span v1.27](https://github.com/open-telemetry/semantic-conventions/blob/v1.27.0/docs/http/http-spans.md) semantic conventions. + Otel configuration should be provided as a map in the `:otel` key of the `Finch.Request.private` map. + See the headings below for examples. + + ### Span name + + The span name can be as follows in order of precedence: + + - `{method}` - default name if no other options are specified + - `{span_name}` - custom span name defined with the `span_name` option + - `{method} {url_template}` - custom span name defined with the `url_template` options + + Example: + ```elixir + Finch.build(:get, url) + |> Finch.Request.put_private(:otel, %{span_name: "custom_span_name"}) + |> Finch.request(HttpFinch) + ``` + + ### Opt-in Semantic Convention Attributes + + Otel SemConv requires users to explicitly opt in for any attribute with a + requirement level of `opt-in` or `experimental`. To ensure compatibility, always use the + SemConv attribute. + + Example: + ```elixir + Finch.build(:get, url) + |> Finch.Request.put_private( + :otel, + %{opt_in_attrs: [SemConv.URLAttributes.url_template()]} + ) + |> Finch.request(HttpFinch) + ``` + + Request and response header attributes are opt-in and can be set with the + `request_header_attrs` and `response_header_attrs` options. Values should be lower-case. + + Example: + ```elixir + Finch.build(:get, url) + |> Finch.Request.put_private(:otel, %{response_header_attrs: ["content-length"]}) + |> Finch.request(HttpFinch) + ``` """ - alias OpenTelemetry.SemanticConventions.Trace + alias OpenTelemetry.SemConv.ErrorAttributes + alias OpenTelemetry.SemConv.Incubating.HTTPAttributes + alias OpenTelemetry.SemConv.Incubating.URLAttributes + alias OpenTelemetry.SemConv.NetworkAttributes + alias OpenTelemetry.SemConv.ServerAttributes + alias OpenTelemetry.SemConv.UserAgentAttributes - require Trace require OpenTelemetry.Tracer + opt_ins = [ + HTTPAttributes.http_request_body_size(), + HTTPAttributes.http_response_body_size(), + NetworkAttributes.network_transport(), + URLAttributes.url_scheme(), + URLAttributes.url_template(), + UserAgentAttributes.user_agent_original() + ] + + @options_schema NimbleOptions.new!( + opt_in_attrs: [ + type: {:list, {:in, opt_ins}}, + default: [], + type_spec: quote(do: opt_in_attrs()), + doc: """ + Opt-in and experimental attributes. + Use semantic conventions library to ensure compatibility, e.g. `[HTTPAttributes.http_request_body_size()]` + + #{Enum.map_join(opt_ins, "\n\n", &" * `#{inspect(&1)}`")} + """ + ], + request_header_attrs: [ + type: {:list, :string}, + default: [], + doc: "List of request headers to add as attributes (lowercase)" + ], + response_header_attrs: [ + type: {:list, :string}, + default: [], + doc: "List of response headers to add as attributes (lowercase)" + ], + span_name: [ + type: {:or, [:atom, nil, :string]}, + default: nil, + doc: "User defined span name override" + ], + url_template: [ + type: {:or, [nil, :string]}, + default: nil, + doc: "URL template to use for span name and opt-in attribute" + ] + ) + + @type opt_in_attr() :: + unquote(HTTPAttributes.http_request_body_size()) + | unquote(HTTPAttributes.http_response_body_size()) + | unquote(NetworkAttributes.network_transport()) + | unquote(URLAttributes.url_scheme()) + | unquote(URLAttributes.url_template()) + | unquote(UserAgentAttributes.user_agent_original()) + + @type opt_in_attrs() :: [opt_in_attr()] + @typedoc "Setup options" @type opts :: [] @@ -41,43 +144,190 @@ defmodule OpentelemetryFinch do duration = measurements.duration end_time = :opentelemetry.timestamp() start_time = end_time - duration - - status = - case meta.result do - {:ok, response} -> response.status - _ -> 0 - end - - url = build_url(meta.request.scheme, meta.request.host, meta.request.port, meta.request.path) - - attributes = %{ - Trace.http_url() => url, - Trace.http_scheme() => meta.request.scheme, - Trace.net_peer_name() => meta.request.host, - Trace.net_peer_port() => meta.request.port, - Trace.http_target() => meta.request.path, - Trace.http_method() => meta.request.method, - Trace.http_status_code() => status - } + otel_config = get_otel_config(meta.request.private) + span_name = span_name(meta.request, otel_config) s = - OpenTelemetry.Tracer.start_span("HTTP #{meta.request.method}", %{ + OpenTelemetry.Tracer.start_span(span_name, %{ start_time: start_time, - attributes: attributes, + attributes: build_attrs(meta, otel_config), kind: :client }) - if meta.result |> elem(0) == :error do - OpenTelemetry.Span.set_status( - s, - OpenTelemetry.status(:error, format_error(meta.result |> elem(1))) + set_span_status(s, meta.result) + + OpenTelemetry.Span.end_span(s) + end + + defp get_otel_config(request_private) do + request_private + |> Map.get(:otel, %{}) + |> NimbleOptions.validate!(@options_schema) + end + + defp span_name(request, %{span_name: span_name}) when is_binary(span_name), + do: "#{request.method} #{span_name}" + + defp span_name(request, %{url_template: url_template}) when is_binary(url_template), + do: "#{request.method} #{url_template}" + + defp span_name(request, _), do: "#{request.method}" + + defp build_attrs(meta, otel_config) do + Map.merge( + build_req_attrs(meta.request, otel_config), + build_resp_attrs(meta.result, otel_config) + ) + end + + defp build_req_attrs(request, otel_config) do + url = + build_url( + request.scheme, + request.host, + request.port, + request.path, + request.query ) + + %{ + HTTPAttributes.http_request_method() => request.method, + ServerAttributes.server_address() => request.host, + ServerAttributes.server_port() => request.port, + URLAttributes.url_full() => url + } + |> add_req_header_attrs(request, otel_config) + |> add_opt_in_req_attrs(request, otel_config) + end + + defp add_opt_in_req_attrs(attrs, request, %{opt_in_attrs: [_ | _] = opt_in_attrs} = otel_config) do + %{ + HTTPAttributes.http_request_body_size() => extract_request_body_size(request), + NetworkAttributes.network_transport() => :tcp, + URLAttributes.url_scheme() => request.scheme, + URLAttributes.url_template() => extract_url_template(otel_config), + UserAgentAttributes.user_agent_original() => extract_user_agent(request) + } + |> Map.take(opt_in_attrs) + |> then(&Map.merge(attrs, &1)) + end + + defp add_opt_in_req_attrs(attrs, _request, _otel_config), do: attrs + + defp build_resp_attrs({:ok, response} = result, otel_config) do + %{ + HTTPAttributes.http_response_status_code() => response.status + } + |> maybe_add_error_type(result) + |> add_resp_header_attrs(response, otel_config) + |> add_opt_in_resp_attrs(response, otel_config) + end + + defp build_resp_attrs({:error, _} = result, _otel_config) do + maybe_add_error_type(%{}, result) + end + + defp add_opt_in_resp_attrs(attrs, response, %{opt_in_attrs: [_ | _] = opt_in_attrs}) do + %{ + HTTPAttributes.http_response_body_size() => extract_response_body_size(response) + } + |> Map.take(opt_in_attrs) + |> then(&Map.merge(attrs, &1)) + end + + defp add_opt_in_resp_attrs(attrs, _response, _otel_config), do: attrs + + defp add_req_header_attrs(attrs, req, otel_config) do + Map.merge( + attrs, + :otel_http.extract_headers_attributes( + :request, + req.headers, + Map.get(otel_config, :request_header_attrs, []) + ) + ) + end + + defp add_resp_header_attrs(attrs, resp, otel_config) do + Map.merge( + attrs, + :otel_http.extract_headers_attributes( + :response, + resp.headers, + Map.get(otel_config, :response_header_attrs, []) + ) + ) + end + + defp extract_request_body_size(request) do + case get_header(request.headers, "content-length") do + [] -> + 0 + + [length_str | _] when is_binary(length_str) -> + String.to_integer(length_str) end + end - OpenTelemetry.Span.end_span(s) + defp extract_url_template(otel_config) do + Map.get(otel_config, :url_template, "") + end + + defp extract_user_agent(request) do + case get_header(request.headers, "user-agent") do + [] -> + "" + + [user_agent | _] -> + user_agent + end + end + + defp extract_response_body_size(response) do + case get_header(response.headers, "content-length") do + [] -> + 0 + + [length_str | _] when is_binary(length_str) -> + String.to_integer(length_str) + end + end + + defp maybe_add_error_type(attrs, {:ok, %{status: status}}) when status >= 400 do + Map.put(attrs, ErrorAttributes.error_type(), to_string(status)) + end + + defp maybe_add_error_type(attrs, {:error, reason}) do + Map.put(attrs, ErrorAttributes.error_type(), format_error(reason)) + end + + defp maybe_add_error_type(attrs, _), do: attrs + + defp set_span_status(span, {:ok, %{status: status}}) when status >= 400 do + OpenTelemetry.Span.set_status(span, OpenTelemetry.status(:error, to_string(status))) + end + + defp set_span_status(span, {:error, reason}) do + OpenTelemetry.Span.set_status(span, OpenTelemetry.status(:error, format_error(reason))) + end + + defp set_span_status(span, _result) do + OpenTelemetry.Span.set_status(span, OpenTelemetry.status(:ok, "")) + end + + defp get_header(headers, header_name) do + headers + |> Enum.filter(fn {k, _} -> k == header_name end) + |> Enum.map(fn {_, v} -> v end) + end + + defp build_url(scheme, host, port, path, query) do + "#{scheme}://#{host}:#{port}#{path}" |> append_query(query) end - defp build_url(scheme, host, port, path), do: "#{scheme}://#{host}:#{port}#{path}" + defp append_query(url, nil), do: url + defp append_query(url, ""), do: url + defp append_query(url, query), do: "#{url}?#{query}" defp format_error(%{__exception__: true} = exception), do: Exception.message(exception) defp format_error(reason), do: inspect(reason) diff --git a/instrumentation/opentelemetry_finch/mix.exs b/instrumentation/opentelemetry_finch/mix.exs index c72f695c..0ab0238b 100644 --- a/instrumentation/opentelemetry_finch/mix.exs +++ b/instrumentation/opentelemetry_finch/mix.exs @@ -59,6 +59,7 @@ defmodule OpentelemetryFinch.MixProject do {:opentelemetry_semantic_conventions, "~> 1.27"}, {:opentelemetry, "~> 1.5", only: [:dev, :test]}, {:opentelemetry_exporter, "~> 1.8", only: [:dev, :test]}, + {:otel_http, "~> 0.2"}, {:ex_doc, "~> 0.38", only: [:dev], runtime: false}, {:finch, "~> 0.19", only: [:dev, :test]}, {:dialyxir, "~> 1.4", only: [:dev, :test], runtime: false}, diff --git a/instrumentation/opentelemetry_finch/mix.lock b/instrumentation/opentelemetry_finch/mix.lock index 1b26dad7..c3accebf 100644 --- a/instrumentation/opentelemetry_finch/mix.lock +++ b/instrumentation/opentelemetry_finch/mix.lock @@ -27,6 +27,7 @@ "opentelemetry_api": {:hex, :opentelemetry_api, "1.4.0", "63ca1742f92f00059298f478048dfb826f4b20d49534493d6919a0db39b6db04", [:mix, :rebar3], [], "hexpm", "3dfbbfaa2c2ed3121c5c483162836c4f9027def469c41578af5ef32589fcfc58"}, "opentelemetry_exporter": {:hex, :opentelemetry_exporter, "1.8.0", "5d546123230771ef4174e37bedfd77e3374913304cd6ea3ca82a2add49cd5d56", [:rebar3], [{:grpcbox, ">= 0.0.0", [hex: :grpcbox, repo: "hexpm", optional: false]}, {:opentelemetry, "~> 1.5.0", [hex: :opentelemetry, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.4.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:tls_certificate_check, "~> 1.18", [hex: :tls_certificate_check, repo: "hexpm", optional: false]}], "hexpm", "a1f9f271f8d3b02b81462a6bfef7075fd8457fdb06adff5d2537df5e2264d9af"}, "opentelemetry_semantic_conventions": {:hex, :opentelemetry_semantic_conventions, "1.27.0", "acd0194a94a1e57d63da982ee9f4a9f88834ae0b31b0bd850815fe9be4bbb45f", [:mix, :rebar3], [], "hexpm", "9681ccaa24fd3d810b4461581717661fd85ff7019b082c2dff89c7d5b1fc2864"}, + "otel_http": {:hex, :otel_http, "0.2.0", "b17385986c7f1b862f5d577f72614ecaa29de40392b7618869999326b9a61d8a", [:rebar3], [], "hexpm", "f2beadf922c8cfeb0965488dd736c95cc6ea8b9efce89466b3904d317d7cc717"}, "plug": {:hex, :plug, "1.15.3", "712976f504418f6dff0a3e554c40d705a9bcf89a7ccef92fc6a5ef8f16a30a97", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "cc4365a3c010a56af402e0809208873d113e9c38c401cabd88027ef4f5c01fd2"}, "plug_cowboy": {:hex, :plug_cowboy, "2.7.0", "3ae9369c60641084363b08fe90267cbdd316df57e3557ea522114b30b63256ea", [:mix], [{:cowboy, "~> 2.7.0 or ~> 2.8.0 or ~> 2.9.0 or ~> 2.10.0", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "d85444fb8aa1f2fc62eabe83bbe387d81510d773886774ebdcb429b3da3c1a4a"}, "plug_crypto": {:hex, :plug_crypto, "2.0.0", "77515cc10af06645abbfb5e6ad7a3e9714f805ae118fa1a70205f80d2d70fe73", [:mix], [], "hexpm", "53695bae57cc4e54566d993eb01074e4d894b65a3766f1c43e2c61a1b0f45ea9"}, diff --git a/instrumentation/opentelemetry_finch/test/opentelemetry_finch_test.exs b/instrumentation/opentelemetry_finch/test/opentelemetry_finch_test.exs index 98b44a65..260afd61 100644 --- a/instrumentation/opentelemetry_finch/test/opentelemetry_finch_test.exs +++ b/instrumentation/opentelemetry_finch/test/opentelemetry_finch_test.exs @@ -7,6 +7,13 @@ defmodule OpentelemetryFinchTest do require OpenTelemetry.Span require Record + alias OpenTelemetry.SemConv.ErrorAttributes + alias OpenTelemetry.SemConv.Incubating.HTTPAttributes + alias OpenTelemetry.SemConv.Incubating.URLAttributes + alias OpenTelemetry.SemConv.NetworkAttributes + alias OpenTelemetry.SemConv.ServerAttributes + alias OpenTelemetry.SemConv.UserAgentAttributes + for {name, spec} <- Record.extract_all(from_lib: "opentelemetry/include/otel_span.hrl") do Record.defrecord(name, spec) end @@ -16,6 +23,10 @@ defmodule OpentelemetryFinchTest do end setup do + OpentelemetryFinch.setup() + + _conn = start_supervised!({Finch, name: HttpFinch}) + :otel_simple_processor.set_exporter(:otel_exporter_pid, self()) OpenTelemetry.Tracer.start_span("test") @@ -29,50 +40,220 @@ defmodule OpentelemetryFinchTest do test "records span on requests", %{bypass: bypass} do Bypass.expect(bypass, fn conn -> Plug.Conn.resp(conn, 200, "") end) - OpentelemetryFinch.setup() - _conn = start_supervised!({Finch, name: HttpFinch}) + url = "#{endpoint_url(bypass.port)}/users/2?token=some-token&array=foo&array=bar" + {:ok, _} = Finch.build(:get, url) |> Finch.request(HttpFinch) - {:ok, _} = Finch.build(:get, endpoint_url(bypass.port)) |> Finch.request(HttpFinch) + assert_receive {:span, + span( + name: "GET", + kind: :client, + attributes: attributes + )} + + attrs = :otel_attributes.map(attributes) + + expected_attrs = [ + {ServerAttributes.server_address(), "localhost"}, + {ServerAttributes.server_port(), bypass.port}, + {HTTPAttributes.http_request_method(), "GET"}, + {URLAttributes.url_full(), url}, + {HTTPAttributes.http_response_status_code(), 200} + ] + + for {attr, val} <- expected_attrs do + assert Map.get(attrs, attr) == val, " expected #{attr} to equal #{val}" + end + end + + test "records span on requests with custom span name", %{bypass: bypass} do + Bypass.expect(bypass, fn conn -> Plug.Conn.resp(conn, 200, "") end) + + otel_config = %{span_name: "/users/:user_id"} + + {:ok, _} = + Finch.build(:get, endpoint_url(bypass.port)) + |> Finch.Request.put_private(:otel, otel_config) + |> Finch.request(HttpFinch) + + assert_receive {:span, span(name: "GET /users/:user_id", kind: :client)} + end + + test "records span on requests with url template", %{bypass: bypass} do + Bypass.expect(bypass, fn conn -> Plug.Conn.resp(conn, 200, "") end) + + otel_config = %{url_template: "/users/:user_id"} + + {:ok, _} = + Finch.build(:get, endpoint_url(bypass.port)) + |> Finch.Request.put_private(:otel, otel_config) + |> Finch.request(HttpFinch) + + assert_receive {:span, span(name: "GET /users/:user_id", kind: :client)} + end + + test "adds opt-in attrs to span when opt_in_attrs is set", %{bypass: bypass} do + Bypass.expect(bypass, fn conn -> Plug.Conn.resp(conn, 200, "") end) + + otel_config = %{ + opt_in_attrs: [ + HTTPAttributes.http_request_body_size(), + HTTPAttributes.http_response_body_size(), + NetworkAttributes.network_transport(), + URLAttributes.url_scheme(), + URLAttributes.url_template(), + UserAgentAttributes.user_agent_original() + ], + url_template: "/users/:user_id" + } + + {:ok, _} = + Finch.build(:get, endpoint_url(bypass.port)) + |> Finch.Request.put_private(:otel, otel_config) + |> Finch.request(HttpFinch) assert_receive {:span, span( - name: "HTTP GET", + name: "GET /users/:user_id", kind: :client, attributes: attributes )} - assert %{ - "net.peer.name": "localhost", - "http.method": "GET", - "http.target": "/", - "http.scheme": :http, - "http.status_code": 200 - } = :otel_attributes.map(attributes) + attrs = :otel_attributes.map(attributes) + + expected_attrs = [ + {ServerAttributes.server_address(), "localhost"}, + {ServerAttributes.server_port(), bypass.port}, + {HTTPAttributes.http_request_method(), "GET"}, + {URLAttributes.url_full(), endpoint_url(bypass.port)}, + {HTTPAttributes.http_response_status_code(), 200}, + {HTTPAttributes.http_request_body_size(), 0}, + {HTTPAttributes.http_response_body_size(), 0}, + {NetworkAttributes.network_transport(), :tcp}, + {URLAttributes.url_scheme(), :http}, + {URLAttributes.url_template(), "/users/:user_id"}, + {UserAgentAttributes.user_agent_original(), ""} + ] + + for {attr, val} <- expected_attrs do + assert Map.get(attrs, attr) == val, " expected #{attr} to equal #{val}" + end end - test "records span on requests failed", %{bypass: _} do - OpentelemetryFinch.setup() + test "adds request and response headers to span when request_header_attrs and response_header_attrs are set", + %{bypass: bypass} do + Bypass.expect(bypass, fn conn -> Plug.Conn.resp(conn, 200, "") end) - _conn = start_supervised!({Finch, name: HttpFinch}) + otel_config = %{ + request_header_attrs: ["authorization"], + response_header_attrs: ["content-length", "server"] + } + + {:ok, _} = + Finch.build(:get, endpoint_url(bypass.port), [{"authorization", "Bearer token"}]) + |> Finch.Request.put_private(:otel, otel_config) + |> Finch.request(HttpFinch) + + assert_receive {:span, + span( + name: "GET", + kind: :client, + attributes: attributes + )} + + attrs = :otel_attributes.map(attributes) + + expected_attrs = [ + {ServerAttributes.server_address(), "localhost"}, + {ServerAttributes.server_port(), bypass.port}, + {HTTPAttributes.http_request_method(), "GET"}, + {URLAttributes.url_full(), endpoint_url(bypass.port)}, + {HTTPAttributes.http_response_status_code(), 200}, + {String.to_atom("#{HTTPAttributes.http_response_header()}.content-length"), ["0"]}, + {String.to_atom("#{HTTPAttributes.http_response_header()}.server"), ["Cowboy"]}, + {String.to_atom("#{HTTPAttributes.http_request_header()}.authorization"), ["Bearer token"]} + ] + + for {attr, val} <- expected_attrs do + assert Map.get(attrs, attr) == val, " expected #{attr} to equal #{val}" + end + end + test "records span on requests failed", %{bypass: _} do {:error, _} = Finch.build(:get, endpoint_url(3333)) |> Finch.request(HttpFinch) assert_receive {:span, span( - name: "HTTP GET", + name: "GET", kind: :client, status: {:status, :error, "connection refused"}, attributes: attributes )} - assert %{ - "net.peer.name": "localhost", - "http.method": "GET", - "http.target": "/", - "http.scheme": :http, - "http.status_code": 0 - } = :otel_attributes.map(attributes) + attrs = :otel_attributes.map(attributes) + + expected_attrs = [ + {ServerAttributes.server_address(), "localhost"}, + {ServerAttributes.server_port(), 3333}, + {HTTPAttributes.http_request_method(), "GET"}, + {URLAttributes.url_full(), "http://localhost:3333/"}, + {ErrorAttributes.error_type(), "connection refused"} + ] + + for {attr, val} <- expected_attrs do + assert Map.get(attrs, attr) == val, " expected #{attr} to equal #{val}" + end + end + + test "records span on request response with 4xx status code", %{bypass: bypass} do + Bypass.expect(bypass, fn conn -> Plug.Conn.resp(conn, 404, "") end) + + {:ok, _} = Finch.build(:get, endpoint_url(bypass.port)) |> Finch.request(HttpFinch) + + assert_receive {:span, + span( + name: "GET", + kind: :client, + status: {:status, :error, "404"}, + attributes: attributes + )} + + attrs = :otel_attributes.map(attributes) + + expected_attrs = [ + {ServerAttributes.server_address(), "localhost"}, + {ServerAttributes.server_port(), bypass.port}, + {HTTPAttributes.http_request_method(), "GET"}, + {URLAttributes.url_full(), endpoint_url(bypass.port)}, + {HTTPAttributes.http_response_status_code(), 404}, + {ErrorAttributes.error_type(), "404"} + ] + + for {attr, val} <- expected_attrs do + assert Map.get(attrs, attr) == val, " expected #{attr} to equal #{val}" + end + end + + test "logs error and doesn't record span if invalid otel option is passed", %{bypass: bypass} do + Bypass.expect(bypass, fn conn -> Plug.Conn.resp(conn, 200, "") end) + + log = + ExUnit.CaptureLog.capture_log(fn -> + Finch.build(:get, endpoint_url(bypass.port)) + |> Finch.Request.put_private(:otel, %{invalid_option: "invalid_value"}) + |> Finch.request(HttpFinch) + end) + + assert log =~ """ + Reason=%NimbleOptions.ValidationError{ + message: \"unknown options [:invalid_option], valid options are: [:opt_in_attrs, :request_header_attrs, :response_header_attrs, :span_name, :url_template]\", + key: [:invalid_option], + value: nil, + keys_path: [] + }\ + """ + + refute_receive {:span, _} end defp endpoint_url(port), do: "http://localhost:#{port}/"