Add a custom OTel filtered sampler

Author: joshk

config/dev.exs

@@ -112,5 +112,3 @@ config :nerves_hub, NervesHub.RateLimit, limit: 10
 
 config :nerves_hub,
   open_for_registrations: true
-
-config :opentelemetry, traces_exporter: :none

config/runtime.exs

@@ -387,26 +387,25 @@ config :sentry,
     ]
   ]
 
-if otlp_endpoint = System.get_env("OTLP_ENDPOINT") do
-  config :opentelemetry, :resource, service: %{name: nerves_hub_app}
+config :opentelemetry, :resource, service: %{name: nerves_hub_app}
 
+if otlp_endpoint = System.get_env("OTLP_ENDPOINT") do
   config :opentelemetry_exporter,
     otlp_protocol: :http_protobuf,
     otlp_endpoint: otlp_endpoint,
     otlp_headers: [{System.get_env("OTLP_AUTH_HEADER"), System.get_env("OTLP_AUTH_HEADER_VALUE")}]
 
-  if otlp_sampler_ratio = System.get_env("OTLP_SAMPLER_RATIO") do
-    config :opentelemetry,
-      sampler:
-        {:parent_based,
-         %{
-           root: {:trace_id_ratio_based, String.to_float(otlp_sampler_ratio)},
-           remote_parent_sampled: :always_on,
-           remote_parent_not_sampled: :always_off,
-           local_parent_sampled: :always_on,
-           local_parent_not_sampled: :always_off
-         }}
-  end
+  otlp_sampler_ratio =
+    if ratio = System.get_env("OTLP_SAMPLER_RATIO") do
+      String.to_float(ratio)
+    else
+      nil
+    end
+
+  config :opentelemetry,
+    sampler: {:parent_based, %{root: {NervesHub.Telemetry.FilteredSampler, otlp_sampler_ratio}}}
+else
+  config :opentelemetry, traces_exporter: :none
 end
 
 if host = System.get_env("STATSD_HOST") do

config/test.exs

@@ -91,5 +91,3 @@ config :phoenix_test, :endpoint, NervesHubWeb.Endpoint
 
 # Initialize plugs at runtime for faster test compilation
 config :phoenix, :plug_init_mode, :runtime
-
-config :opentelemetry, traces_exporter: :none

lib/nerves_hub/application.ex

@@ -12,18 +12,7 @@ defmodule NervesHub.Application do
         raise "fwup could not be found in the $PATH. This is a requirement of NervesHubWeb and cannot start otherwise"
     end
 
-    if System.get_env("ECTO_IPV6") do
-      :httpc.set_option(:ipfamily, :inet6fb4)
-    end
-
-    :ok = OpentelemetryBandit.setup()
-    :ok = OpentelemetryPhoenix.setup(adapter: :bandit)
-    :ok = OpentelemetryOban.setup(trace: [:jobs])
-
-    :ok =
-      NervesHub.Repo.config()
-      |> Keyword.fetch!(:telemetry_prefix)
-      |> OpentelemetryEcto.setup(db_statement: :enabled)
+    setup_open_telemetry()
 
     _ =
       :logger.add_handler(:my_sentry_handler, Sentry.LoggerHandler, %{
@@ -60,6 +49,23 @@ defmodule NervesHub.Application do
     Supervisor.start_link(children, opts)
   end
 
+  defp setup_open_telemetry() do
+    if System.get_env("ECTO_IPV6") do
+      :httpc.set_option(:ipfamily, :inet6fb4)
+    end
+
+    :ok = OpentelemetryBandit.setup()
+    :ok = OpentelemetryPhoenix.setup(adapter: :bandit)
+    :ok = OpentelemetryOban.setup(trace: [:jobs])
+
+    :ok =
+      NervesHub.Repo.config()
+      |> Keyword.fetch!(:telemetry_prefix)
+      |> OpentelemetryEcto.setup(db_statement: :enabled)
+
+    :ok
+  end
+
   def config_change(changed, _new, removed) do
     NervesHubWeb.Endpoint.config_change(changed, removed)
     :ok

lib/nerves_hub/telemetry/filtered_sampler.ex

@@ -0,0 +1,88 @@
+defmodule NervesHub.Telemetry.FilteredSampler do
+  # Inspired by https://arathunku.com/b/2024/notes-on-adding-opentelemetry-to-an-elixir-app/
+
+  # TODO: Add ratio sampling support
+
+  require OpenTelemetry.Tracer, as: Tracer
+  require Logger
+
+  @behaviour :otel_sampler
+
+  @ignored_static_paths ~r/^\/(assets|fonts|images)\/.*/
+
+  @ignored_url_paths [
+    "/status/alive",
+    "/favicon.ico",
+    "/phoenix/live_reload/socket/websocket",
+    # "/live/websocket",
+    "/"
+  ]
+
+  @ignored_span_names [
+    "Channels.DeviceSocket.heartbeat",
+    "nerves_hub.repo.query:schema_migrations"
+  ]
+
+  @impl :otel_sampler
+  def setup(probability \\ nil) do
+    if probability do
+      [ratio_sampler_config: :otel_sampler_trace_id_ratio_based.setup(probability)]
+    else
+      []
+    end
+  end
+
+  @impl :otel_sampler
+  def description(_sampler_config), do: "NervesHub.Sampler"
+
+  @impl :otel_sampler
+  def should_sample(
+        ctx,
+        trace_id,
+        links,
+        span_name,
+        span_kind,
+        attributes,
+        sampler_config
+      ) do
+    result = drop_trace?(span_name, attributes)
+
+    tracestate = Tracer.current_span_ctx(ctx) |> OpenTelemetry.Span.tracestate()
+
+    case result do
+      true ->
+        {:drop, [], tracestate}
+
+      false ->
+        if config = sampler_config[:ratio_sampler_config] do
+          :otel_sampler_trace_id_ratio_based.should_sample(
+            ctx,
+            trace_id,
+            links,
+            span_name,
+            span_kind,
+            attributes,
+            config
+          )
+        else
+          {:record_and_sample, [], tracestate}
+        end
+    end
+  end
+
+  def drop_trace?(span_name, attributes) do
+    cond do
+      Enum.member?(@ignored_span_names, span_name) ->
+        true
+
+      span_name == :GET && Enum.member?(@ignored_url_paths, attributes[:"url.path"]) ->
+        true
+
+      span_name == :GET && (attributes[:"url.path"] || "") =~ @ignored_static_paths ->
+        true
+
+      true ->
+        false
+    end
+  end
+end