From a362ae1efda8cdee679237f405a55e95b2ba24a8 Mon Sep 17 00:00:00 2001
From: Josh Kalderimis <josh.kalderimis@gmail.com>
Date: Sat, 26 Oct 2024 12:29:44 +1300
Subject: [PATCH 01/14] Add support for OpenTelemetry tracing

---
 config/dev.exs                             |   2 +
 config/runtime.exs                         |  22 +++++
 lib/nerves_hub/application.ex              |  13 +++
 lib/nerves_hub/deployments/orchestrator.ex | 102 +++++++++++----------
 lib/nerves_hub_web/live/devices/index.ex   |  14 ++-
 mix.exs                                    |  12 +++
 mix.lock                                   |  17 ++++
 7 files changed, 128 insertions(+), 54 deletions(-)

diff --git a/config/dev.exs b/config/dev.exs
index 90f729676..3e91c693c 100644
--- a/config/dev.exs
+++ b/config/dev.exs
@@ -112,3 +112,5 @@ config :nerves_hub, NervesHub.RateLimit, limit: 10
 
 config :nerves_hub,
   open_for_registrations: true
+
+config :opentelemetry, traces_exporter: :none
diff --git a/config/runtime.exs b/config/runtime.exs
index 4115f7f8d..e1c3e91e1 100644
--- a/config/runtime.exs
+++ b/config/runtime.exs
@@ -387,6 +387,28 @@ config :sentry,
     ]
   ]
 
+if otlp_endpoint = System.get_env("OTLP_ENDPOINT") do
+  config :opentelemetry, :resource, service: %{name: nerves_hub_app}
+
+  config :opentelemetry_exporter,
+    otlp_protocol: :http_protobuf,
+    otlp_endpoint: otlp_endpoint,
+    otlp_headers: [{System.get_env("OTLP_AUTH_HEADER"), System.get_env("OTLP_AUTH_HEADER_VALUE")}]
+
+  if otlp_sampler_ratio = System.get_env("OTLP_SAMPLER_RATIO") do
+    config :opentelemetry,
+      sampler:
+        {:parent_based,
+         %{
+           root: {:trace_id_ratio_based, String.to_float(otlp_sampler_ratio)},
+           remote_parent_sampled: :always_on,
+           remote_parent_not_sampled: :always_off,
+           local_parent_sampled: :always_on,
+           local_parent_not_sampled: :always_off
+         }}
+  end
+end
+
 if host = System.get_env("STATSD_HOST") do
   config :nerves_hub, :statsd,
     host: System.get_env("STATSD_HOST"),
diff --git a/lib/nerves_hub/application.ex b/lib/nerves_hub/application.ex
index 8d107b846..0c9efb4ab 100644
--- a/lib/nerves_hub/application.ex
+++ b/lib/nerves_hub/application.ex
@@ -12,6 +12,19 @@ defmodule NervesHub.Application do
         raise "fwup could not be found in the $PATH. This is a requirement of NervesHubWeb and cannot start otherwise"
     end
 
+    if System.get_env("ECTO_IPV6") do
+      :httpc.set_option(:ipfamily, :inet6fb4)
+    end
+
+    :ok = OpentelemetryBandit.setup()
+    :ok = OpentelemetryPhoenix.setup(adapter: :bandit)
+    :ok = OpentelemetryOban.setup(trace: [:jobs])
+
+    :ok =
+      NervesHub.Repo.config()
+      |> Keyword.fetch!(:telemetry_prefix)
+      |> OpentelemetryEcto.setup(db_statement: :enabled)
+
     _ =
       :logger.add_handler(:my_sentry_handler, Sentry.LoggerHandler, %{
         config: %{metadata: [:file, :line]}
diff --git a/lib/nerves_hub/deployments/orchestrator.ex b/lib/nerves_hub/deployments/orchestrator.ex
index b72aeab82..557a1fcee 100644
--- a/lib/nerves_hub/deployments/orchestrator.ex
+++ b/lib/nerves_hub/deployments/orchestrator.ex
@@ -12,6 +12,8 @@ defmodule NervesHub.Deployments.Orchestrator do
 
   require Logger
 
+  require OpenTelemetry.Tracer, as: Tracer
+
   alias NervesHub.Devices
   alias NervesHub.Devices.Device
   alias NervesHub.Repo
@@ -49,57 +51,59 @@ defmodule NervesHub.Deployments.Orchestrator do
   was successful, and the process is repeated.
   """
   def trigger_update(deployment) do
-    :telemetry.execute([:nerves_hub, :deployment, :trigger_update], %{count: 1})
-
-    match_conditions = [
-      {:and, {:==, {:map_get, :deployment_id, :"$1"}, deployment.id},
-       {:==, {:map_get, :updating, :"$1"}, false},
-       {:==, {:map_get, :updates_enabled, :"$1"}, true},
-       {:"/=", {:map_get, :firmware_uuid, :"$1"}, deployment.firmware.uuid}}
-    ]
-
-    match_return = %{
-      device_id: {:element, 1, :"$_"},
-      pid: {:element, 1, {:element, 2, :"$_"}},
-      firmware_uuid: {:map_get, :firmware_uuid, {:element, 2, {:element, 2, :"$_"}}}
-    }
-
-    devices =
-      Registry.select(NervesHub.Devices.Registry, [
-        {{:_, :_, :"$1"}, match_conditions, [match_return]}
-      ])
-
-    # Get a rough count of devices to update
-    count = deployment.concurrent_updates - Devices.count_inflight_updates_for(deployment)
-    # Just in case inflight goes higher than concurrent, limit it to 0
-    count = max(count, 0)
-
-    # use a reduce to bounce out early?
-    # limit the number of devices to 5 minutes / 500ms?
-
-    devices
-    |> Enum.take(count)
-    |> Enum.each(fn %{device_id: device_id, pid: pid} ->
-      :telemetry.execute([:nerves_hub, :deployment, :trigger_update, :device], %{count: 1})
-
-      device = %Device{id: device_id}
-
-      # Check again because other nodes are processing at the same time
-      if Devices.count_inflight_updates_for(deployment) < deployment.concurrent_updates do
-        case Devices.told_to_update(device, deployment) do
-          {:ok, inflight_update} ->
-            send(pid, {"deployments/update", inflight_update})
-
-          :error ->
-            Logger.error(
-              "An inflight update could not be created or found for the device #{device.identifier} (#{device.id})"
-            )
+    Tracer.with_span "NervesHub.Deployments.Orchestrator.trigger_update" do
+      :telemetry.execute([:nerves_hub, :deployment, :trigger_update], %{count: 1})
+
+      match_conditions = [
+        {:and, {:==, {:map_get, :deployment_id, :"$1"}, deployment.id},
+         {:==, {:map_get, :updating, :"$1"}, false},
+         {:==, {:map_get, :updates_enabled, :"$1"}, true},
+         {:"/=", {:map_get, :firmware_uuid, :"$1"}, deployment.firmware.uuid}}
+      ]
+
+      match_return = %{
+        device_id: {:element, 1, :"$_"},
+        pid: {:element, 1, {:element, 2, :"$_"}},
+        firmware_uuid: {:map_get, :firmware_uuid, {:element, 2, {:element, 2, :"$_"}}}
+      }
+
+      devices =
+        Registry.select(NervesHub.Devices.Registry, [
+          {{:_, :_, :"$1"}, match_conditions, [match_return]}
+        ])
+
+      # Get a rough count of devices to update
+      count = deployment.concurrent_updates - Devices.count_inflight_updates_for(deployment)
+      # Just in case inflight goes higher than concurrent, limit it to 0
+      count = max(count, 0)
+
+      # use a reduce to bounce out early?
+      # limit the number of devices to 5 minutes / 500ms?
+
+      devices
+      |> Enum.take(count)
+      |> Enum.each(fn %{device_id: device_id, pid: pid} ->
+        :telemetry.execute([:nerves_hub, :deployment, :trigger_update, :device], %{count: 1})
+
+        device = %Device{id: device_id}
+
+        # Check again because other nodes are processing at the same time
+        if Devices.count_inflight_updates_for(deployment) < deployment.concurrent_updates do
+          case Devices.told_to_update(device, deployment) do
+            {:ok, inflight_update} ->
+              send(pid, {"deployments/update", inflight_update})
+
+            :error ->
+              Logger.error(
+                "An inflight update could not be created or found for the device #{device.identifier} (#{device.id})"
+              )
+          end
         end
-      end
 
-      # Slow the update a bit to allow for concurrent nodes
-      Process.sleep(500)
-    end)
+        # Slow the update a bit to allow for concurrent nodes
+        Process.sleep(500)
+      end)
+    end
   end
 
   def init(deployment) do
diff --git a/lib/nerves_hub_web/live/devices/index.ex b/lib/nerves_hub_web/live/devices/index.ex
index 2e3e0e54e..12067787d 100644
--- a/lib/nerves_hub_web/live/devices/index.ex
+++ b/lib/nerves_hub_web/live/devices/index.ex
@@ -3,6 +3,8 @@ defmodule NervesHubWeb.Live.Devices.Index do
 
   require Logger
 
+  require OpenTelemetry.Tracer, as: Tracer
+
   alias NervesHub.AuditLogs
   alias NervesHub.Devices
   alias NervesHub.Devices.Alarms
@@ -356,12 +358,14 @@ defmodule NervesHubWeb.Live.Devices.Index do
   end
 
   def handle_info(:refresh_device_list, socket) do
-    Process.send_after(self(), :refresh_device_list, @list_refresh_time)
+    Tracer.with_span "NervesHubWeb.Live.Devices.Index.refresh_device_list" do
+      Process.send_after(self(), :refresh_device_list, @list_refresh_time)
 
-    if socket.assigns.paginate_opts.total_pages == 1 do
-      {:noreply, assign_display_devices(socket)}
-    else
-      {:noreply, socket}
+      if socket.assigns.paginate_opts.total_pages == 1 do
+        {:noreply, assign_display_devices(socket)}
+      else
+        {:noreply, socket}
+      end
     end
   end
 
diff --git a/mix.exs b/mix.exs
index 2587bd461..f032ec4f1 100644
--- a/mix.exs
+++ b/mix.exs
@@ -18,6 +18,8 @@ defmodule NervesHub.MixProject do
           steps: [:assemble],
           include_executables_for: [:unix],
           applications: [
+            opentelemetry_exporter: :permanent,
+            opentelemetry: :temporary,
             nerves_hub: :permanent
           ]
         ]
@@ -95,6 +97,16 @@ defmodule NervesHub.MixProject do
       {:mox, "~> 1.0", only: [:test, :dev]},
       {:nimble_csv, "~> 1.1"},
       {:oban, "~> 2.11"},
+      {:opentelemetry_exporter, "~> 1.8"},
+      {:opentelemetry, "~> 1.5"},
+      {:opentelemetry_api, "~> 1.4"},
+      {:opentelemetry_ecto, "~> 1.2"},
+      {:opentelemetry_phoenix, "~> 2.0.0-rc.1 "},
+      {:opentelemetry_oban, "~> 1.0",
+       git: "https://github.com/joshk/opentelemetry-erlang-contrib",
+       branch: "update-obans-semantic-conventions",
+       subdir: "instrumentation/opentelemetry_oban"},
+      {:opentelemetry_bandit, "~> 0.2.0-rc.1"},
       {:phoenix, "~> 1.7.0"},
       {:phoenix_ecto, "~> 4.0"},
       {:phoenix_html, "~> 3.3.1", override: true},
diff --git a/mix.lock b/mix.lock
index 9cf4484b2..45807f002 100644
--- a/mix.lock
+++ b/mix.lock
@@ -1,10 +1,12 @@
 %{
+  "acceptor_pool": {:hex, :acceptor_pool, "1.0.0", "43c20d2acae35f0c2bcd64f9d2bde267e459f0f3fd23dab26485bf518c281b21", [:rebar3], [], "hexpm", "0cbcd83fdc8b9ad2eee2067ef8b91a14858a5883cb7cd800e6fcd5803e158788"},
   "assert_eventually": {:hex, :assert_eventually, "1.0.0", "f1539f28ba3ffa99a712433c77723c7103986932aa341d05eee94c333a920d15", [:mix], [{:ex_doc, ">= 0.0.0", [hex: :ex_doc, repo: "hexpm", optional: true]}], "hexpm", "c658ac4103c8bd82d0cf72a2fdb77477ba3fbc6b15228c5c801003d239625c69"},
   "bandit": {:hex, :bandit, "1.5.7", "6856b1e1df4f2b0cb3df1377eab7891bec2da6a7fd69dc78594ad3e152363a50", [:mix], [{:hpax, "~> 1.0.0", [hex: :hpax, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:thousand_island, "~> 1.0", [hex: :thousand_island, repo: "hexpm", optional: false]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "f2dd92ae87d2cbea2fa9aa1652db157b6cba6c405cb44d4f6dd87abba41371cd"},
   "base62": {:hex, :base62, "1.2.2", "85c6627eb609317b70f555294045895ffaaeb1758666ab9ef9ca38865b11e629", [:mix], [{:custom_base, "~> 0.2.1", [hex: :custom_base, repo: "hexpm", optional: false]}], "hexpm", "d41336bda8eaa5be197f1e4592400513ee60518e5b9f4dcf38f4b4dae6f377bb"},
   "bcrypt_elixir": {:hex, :bcrypt_elixir, "3.2.0", "feab711974beba4cb348147170346fe097eea2e840db4e012a145e180ed4ab75", [:make, :mix], [{:comeonin, "~> 5.3", [hex: :comeonin, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "563e92a6c77d667b19c5f4ba17ab6d440a085696bdf4c68b9b0f5b30bc5422b8"},
   "castore": {:hex, :castore, "1.0.10", "43bbeeac820f16c89f79721af1b3e092399b3a1ecc8df1a472738fd853574911", [:mix], [], "hexpm", "1b0b7ea14d889d9ea21202c43a4fa015eb913021cb535e8ed91946f4b77a8848"},
   "certifi": {:hex, :certifi, "2.12.0", "2d1cca2ec95f59643862af91f001478c9863c2ac9cb6e2f89780bfd8de987329", [:rebar3], [], "hexpm", "ee68d85df22e554040cdb4be100f33873ac6051387baf6a8f6ce82272340ff1c"},
+  "chatterbox": {:hex, :ts_chatterbox, "0.15.1", "5cac4d15dd7ad61fc3c4415ce4826fc563d4643dee897a558ec4ea0b1c835c9c", [:rebar3], [{:hpack, "~> 0.3.0", [hex: :hpack_erl, repo: "hexpm", optional: false]}], "hexpm", "4f75b91451338bc0da5f52f3480fa6ef6e3a2aeecfc33686d6b3d0a0948f31aa"},
   "circular_buffer": {:hex, :circular_buffer, "0.4.1", "477f370fd8cfe1787b0a1bade6208bbd274b34f1610e41f1180ba756a7679839", [:mix], [], "hexpm", "633ef2e059dde0d7b89bbab13b1da9d04c6685e80e68fbdf41282d4fae746b72"},
   "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"},
   "comeonin": {:hex, :comeonin, "5.5.0", "364d00df52545c44a139bad919d7eacb55abf39e86565878e17cebb787977368", [:mix], [], "hexpm", "6287fc3ba0aad34883cbe3f7949fc1d1e738e5ccdce77165bc99490aa69f47fb"},
@@ -31,6 +33,7 @@
   "gettext": {:hex, :gettext, "0.26.2", "5978aa7b21fada6deabf1f6341ddba50bc69c999e812211903b169799208f2a8", [:mix], [{:expo, "~> 0.5.1 or ~> 1.0", [hex: :expo, repo: "hexpm", optional: false]}], "hexpm", "aa978504bcf76511efdc22d580ba08e2279caab1066b76bb9aa81c4a1e0a32a5"},
   "hackney": {:hex, :hackney, "1.20.1", "8d97aec62ddddd757d128bfd1df6c5861093419f8f7a4223823537bad5d064e2", [:rebar3], [{:certifi, "~> 2.12.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "fe9094e5f1a2a2c0a7d10918fee36bfec0ec2a979994cff8cfe8058cd9af38e3"},
   "hlclock": {:hex, :hlclock, "1.0.0", "7a72fc7a20a9382499216227edf97a8b118e21fc3fcad0e81b8d10c616ce1431", [:mix], [], "hexpm", "d3f994336a7fcbc68bf08b14b2101b61e57bef82c032a6e05c1cdc753612c941"},
+  "hpack": {:hex, :hpack_erl, "0.3.0", "2461899cc4ab6a0ef8e970c1661c5fc6a52d3c25580bc6dd204f84ce94669926", [:rebar3], [], "hexpm", "d6137d7079169d8c485c6962dfe261af5b9ef60fbc557344511c1e65e3d95fb0"},
   "hpax": {:hex, :hpax, "1.0.0", "28dcf54509fe2152a3d040e4e3df5b265dcb6cb532029ecbacf4ce52caea3fd2", [:mix], [], "hexpm", "7f1314731d711e2ca5fdc7fd361296593fc2542570b3105595bb0bc6d0fad601"},
   "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"},
   "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"},
@@ -50,6 +53,18 @@
   "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"},
   "nimble_strftime": {:hex, :nimble_strftime, "0.1.1", "b988184d1bd945bc139b2c27dd00a6c0774ec94f6b0b580083abd62d5d07818b", [:mix], [], "hexpm", "89e599c9b8b4d1203b7bb5c79eb51ef7c6a28fbc6228230b312f8b796310d755"},
   "oban": {:hex, :oban, "2.18.3", "1608c04f8856c108555c379f2f56bc0759149d35fa9d3b825cb8a6769f8ae926", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "36ca6ca84ef6518f9c2c759ea88efd438a3c81d667ba23b02b062a0aa785475e"},
+  "opentelemetry": {:hex, :opentelemetry, "1.5.0", "7dda6551edfc3050ea4b0b40c0d2570423d6372b97e9c60793263ef62c53c3c2", [:rebar3], [{:opentelemetry_api, "~> 1.4", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}], "hexpm", "cdf4f51d17b592fc592b9a75f86a6f808c23044ba7cf7b9534debbcc5c23b0ee"},
+  "opentelemetry_api": {:hex, :opentelemetry_api, "1.4.0", "63ca1742f92f00059298f478048dfb826f4b20d49534493d6919a0db39b6db04", [:mix, :rebar3], [], "hexpm", "3dfbbfaa2c2ed3121c5c483162836c4f9027def469c41578af5ef32589fcfc58"},
+  "opentelemetry_bandit": {:hex, :opentelemetry_bandit, "0.2.0-rc.1", "3b78e4a472c20102af2d6852a04a890a8780de26feda560a63283045bc9088f5", [:mix], [{:nimble_options, "~> 1.1", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.3", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:opentelemetry_semantic_conventions, "~> 1.27", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: false]}, {:otel_http, "~> 0.2", [hex: :otel_http, repo: "hexpm", optional: false]}, {:plug, ">= 1.15.0", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.2", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9755c38e8b952204299f568cc30b5462851340c8b4b50702614ca403275b573a"},
+  "opentelemetry_ecto": {:hex, :opentelemetry_ecto, "1.2.0", "2382cb47ddc231f953d3b8263ed029d87fbf217915a1da82f49159d122b64865", [:mix], [{:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:opentelemetry_process_propagator, "~> 0.2", [hex: :opentelemetry_process_propagator, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "70dfa2e79932e86f209df00e36c980b17a32f82d175f0068bf7ef9a96cf080cf"},
+  "opentelemetry_exporter": {:hex, :opentelemetry_exporter, "1.8.0", "5d546123230771ef4174e37bedfd77e3374913304cd6ea3ca82a2add49cd5d56", [:rebar3], [{:grpcbox, ">= 0.0.0", [hex: :grpcbox, repo: "hexpm", optional: false]}, {:opentelemetry, "~> 1.5.0", [hex: :opentelemetry, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.4.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:tls_certificate_check, "~> 1.18", [hex: :tls_certificate_check, repo: "hexpm", optional: false]}], "hexpm", "a1f9f271f8d3b02b81462a6bfef7075fd8457fdb06adff5d2537df5e2264d9af"},
+  "opentelemetry_liveview": {:hex, :opentelemetry_liveview, "1.0.0-rc.4", "52915a83809100f31f7b6ea42e00b964a66032b75cc56e5b4cbcf7e21d4a45da", [:mix], [{:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:opentelemetry_telemetry, "~> 1.0.0-beta.7", [hex: :opentelemetry_telemetry, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e06ab69da7ee46158342cac42f1c22886bdeab53e8d8c4e237c3b3c2cf7b815d"},
+  "opentelemetry_oban": {:git, "https://github.com/joshk/opentelemetry-erlang-contrib", "f9b2a59870865fe7011edb65455b849e0c73547e", [branch: "update-obans-semantic-conventions", subdir: "instrumentation/opentelemetry_oban"]},
+  "opentelemetry_phoenix": {:hex, :opentelemetry_phoenix, "2.0.0-rc.1", "252f918a90f6aad4ea76133e03d9421b8014712d3d9186bd2bc54311563b282f", [:mix], [{:nimble_options, "~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.4", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:opentelemetry_process_propagator, "~> 0.3", [hex: :opentelemetry_process_propagator, repo: "hexpm", optional: false]}, {:opentelemetry_semantic_conventions, "~> 1.27", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: false]}, {:opentelemetry_telemetry, "~> 1.1", [hex: :opentelemetry_telemetry, repo: "hexpm", optional: false]}, {:otel_http, "~> 0.2", [hex: :otel_http, repo: "hexpm", optional: false]}, {:plug, ">= 1.11.0", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "74accdfd9f3757b802064f267ecbc5208a97fcb54d100e8e834b56fe09e64a8e"},
+  "opentelemetry_process_propagator": {:hex, :opentelemetry_process_propagator, "0.3.0", "ef5b2059403a1e2b2d2c65914e6962e56371570b8c3ab5323d7a8d3444fb7f84", [:mix, :rebar3], [{:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}], "hexpm", "7243cb6de1523c473cba5b1aefa3f85e1ff8cc75d08f367104c1e11919c8c029"},
+  "opentelemetry_semantic_conventions": {:hex, :opentelemetry_semantic_conventions, "1.27.0", "acd0194a94a1e57d63da982ee9f4a9f88834ae0b31b0bd850815fe9be4bbb45f", [:mix, :rebar3], [], "hexpm", "9681ccaa24fd3d810b4461581717661fd85ff7019b082c2dff89c7d5b1fc2864"},
+  "opentelemetry_telemetry": {:hex, :opentelemetry_telemetry, "1.1.2", "410ab4d76b0921f42dbccbe5a7c831b8125282850be649ee1f70050d3961118a", [:mix, :rebar3], [{:opentelemetry_api, "~> 1.3", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.1", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "641ab469deb181957ac6d59bce6e1321d5fe2a56df444fc9c19afcad623ab253"},
+  "otel_http": {:hex, :otel_http, "0.2.0", "b17385986c7f1b862f5d577f72614ecaa29de40392b7618869999326b9a61d8a", [:rebar3], [], "hexpm", "f2beadf922c8cfeb0965488dd736c95cc6ea8b9efce89466b3904d317d7cc717"},
   "parse_trans": {:hex, :parse_trans, "3.4.1", "6e6aa8167cb44cc8f39441d05193be6e6f4e7c2946cb2759f015f8c56b76e5ff", [:rebar3], [], "hexpm", "620a406ce75dada827b82e453c19cf06776be266f5a67cff34e1ef2cbb60e49a"},
   "phoenix": {:hex, :phoenix, "1.7.14", "a7d0b3f1bc95987044ddada111e77bd7f75646a08518942c72a8440278ae7825", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.7", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "c7859bc56cc5dfef19ecfc240775dae358cbaa530231118a9e014df392ace61a"},
   "phoenix_ecto": {:hex, :phoenix_ecto, "4.6.3", "f686701b0499a07f2e3b122d84d52ff8a31f5def386e03706c916f6feddf69ef", [:mix], [{:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.1", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "909502956916a657a197f94cc1206d9a65247538de8a5e186f7537c895d95764"},
@@ -80,8 +95,10 @@
   "telemetry_metrics": {:hex, :telemetry_metrics, "1.0.0", "29f5f84991ca98b8eb02fc208b2e6de7c95f8bb2294ef244a176675adc7775df", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f23713b3847286a534e005126d4c959ebcca68ae9582118ce436b521d1d47d5d"},
   "telemetry_metrics_statsd": {:hex, :telemetry_metrics_statsd, "0.7.1", "3502235bb5b35ce50d608bf0f34369ef76eb92a4dbc8708c7e8780ca0da2d53e", [:mix], [{:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "06338d9dc3b4a202f11a6e706fd3feba4c46100d0aca23688dea0b8f801c361f"},
   "telemetry_poller": {:hex, :telemetry_poller, "1.1.0", "58fa7c216257291caaf8d05678c8d01bd45f4bdbc1286838a28c4bb62ef32999", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9eb9d9cbfd81cbd7cdd24682f8711b6e2b691289a0de6826e58452f28c103c8f"},
+  "telemetry_registry": {:hex, :telemetry_registry, "0.3.2", "701576890320be6428189bff963e865e8f23e0ff3615eade8f78662be0fc003c", [:mix, :rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e7ed191eb1d115a3034af8e1e35e4e63d5348851d556646d46ca3d1b4e16bab9"},
   "thousand_island": {:hex, :thousand_island, "1.3.5", "6022b6338f1635b3d32406ff98d68b843ba73b3aa95cfc27154223244f3a6ca5", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "2be6954916fdfe4756af3239fb6b6d75d0b8063b5df03ba76fd8a4c87849e180"},
   "timex": {:hex, :timex, "3.7.11", "bb95cb4eb1d06e27346325de506bcc6c30f9c6dea40d1ebe390b262fad1862d1", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.20", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 1.1", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "8b9024f7efbabaf9bd7aa04f65cf8dcd7c9818ca5737677c7b76acbc6a94d1aa"},
+  "tls_certificate_check": {:hex, :tls_certificate_check, "1.24.0", "d00e2887551ff8cdae4d0340d90d9fcbc4943c7b5f49d32ed4bc23aff4db9a44", [:rebar3], [{:ssl_verify_fun, "~> 1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "90b25a58ee433d91c17f036d4d354bf8859a089bfda60e68a86f8eecae45ef1b"},
   "tzdata": {:hex, :tzdata, "1.1.2", "45e5f1fcf8729525ec27c65e163be5b3d247ab1702581a94674e008413eef50b", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "cec7b286e608371602318c414f344941d5eb0375e14cfdab605cca2fe66cba8b"},
   "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"},
   "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"},

From 4c5f21deece6536e52da1ef5829d503094f7abee Mon Sep 17 00:00:00 2001
From: Josh Kalderimis <josh.kalderimis@gmail.com>
Date: Sat, 26 Oct 2024 14:07:01 +1300
Subject: [PATCH 02/14] Use `open_telemetry_decorator`

---
 lib/nerves_hub/deployments/orchestrator.ex | 104 ++++++++++-----------
 mix.exs                                    |   1 +
 mix.lock                                   |   2 +
 3 files changed, 54 insertions(+), 53 deletions(-)

diff --git a/lib/nerves_hub/deployments/orchestrator.ex b/lib/nerves_hub/deployments/orchestrator.ex
index 557a1fcee..d3a2139ae 100644
--- a/lib/nerves_hub/deployments/orchestrator.ex
+++ b/lib/nerves_hub/deployments/orchestrator.ex
@@ -9,11 +9,10 @@ defmodule NervesHub.Deployments.Orchestrator do
   """
 
   use GenServer
+  use OpenTelemetryDecorator
 
   require Logger
 
-  require OpenTelemetry.Tracer, as: Tracer
-
   alias NervesHub.Devices
   alias NervesHub.Devices.Device
   alias NervesHub.Repo
@@ -50,60 +49,59 @@ defmodule NervesHub.Deployments.Orchestrator do
   As devices update and reconnect, the new orchestrator is told that the update
   was successful, and the process is repeated.
   """
+  @decorate with_span("Deployments.Orchestrator.trigger_update")
   def trigger_update(deployment) do
-    Tracer.with_span "NervesHub.Deployments.Orchestrator.trigger_update" do
-      :telemetry.execute([:nerves_hub, :deployment, :trigger_update], %{count: 1})
-
-      match_conditions = [
-        {:and, {:==, {:map_get, :deployment_id, :"$1"}, deployment.id},
-         {:==, {:map_get, :updating, :"$1"}, false},
-         {:==, {:map_get, :updates_enabled, :"$1"}, true},
-         {:"/=", {:map_get, :firmware_uuid, :"$1"}, deployment.firmware.uuid}}
-      ]
-
-      match_return = %{
-        device_id: {:element, 1, :"$_"},
-        pid: {:element, 1, {:element, 2, :"$_"}},
-        firmware_uuid: {:map_get, :firmware_uuid, {:element, 2, {:element, 2, :"$_"}}}
-      }
-
-      devices =
-        Registry.select(NervesHub.Devices.Registry, [
-          {{:_, :_, :"$1"}, match_conditions, [match_return]}
-        ])
-
-      # Get a rough count of devices to update
-      count = deployment.concurrent_updates - Devices.count_inflight_updates_for(deployment)
-      # Just in case inflight goes higher than concurrent, limit it to 0
-      count = max(count, 0)
-
-      # use a reduce to bounce out early?
-      # limit the number of devices to 5 minutes / 500ms?
-
-      devices
-      |> Enum.take(count)
-      |> Enum.each(fn %{device_id: device_id, pid: pid} ->
-        :telemetry.execute([:nerves_hub, :deployment, :trigger_update, :device], %{count: 1})
-
-        device = %Device{id: device_id}
-
-        # Check again because other nodes are processing at the same time
-        if Devices.count_inflight_updates_for(deployment) < deployment.concurrent_updates do
-          case Devices.told_to_update(device, deployment) do
-            {:ok, inflight_update} ->
-              send(pid, {"deployments/update", inflight_update})
-
-            :error ->
-              Logger.error(
-                "An inflight update could not be created or found for the device #{device.identifier} (#{device.id})"
-              )
-          end
+    :telemetry.execute([:nerves_hub, :deployment, :trigger_update], %{count: 1})
+
+    match_conditions = [
+      {:and, {:==, {:map_get, :deployment_id, :"$1"}, deployment.id},
+       {:==, {:map_get, :updating, :"$1"}, false},
+       {:==, {:map_get, :updates_enabled, :"$1"}, true},
+       {:"/=", {:map_get, :firmware_uuid, :"$1"}, deployment.firmware.uuid}}
+    ]
+
+    match_return = %{
+      device_id: {:element, 1, :"$_"},
+      pid: {:element, 1, {:element, 2, :"$_"}},
+      firmware_uuid: {:map_get, :firmware_uuid, {:element, 2, {:element, 2, :"$_"}}}
+    }
+
+    devices =
+      Registry.select(NervesHub.Devices.Registry, [
+        {{:_, :_, :"$1"}, match_conditions, [match_return]}
+      ])
+
+    # Get a rough count of devices to update
+    count = deployment.concurrent_updates - Devices.count_inflight_updates_for(deployment)
+    # Just in case inflight goes higher than concurrent, limit it to 0
+    count = max(count, 0)
+
+    # use a reduce to bounce out early?
+    # limit the number of devices to 5 minutes / 500ms?
+
+    devices
+    |> Enum.take(count)
+    |> Enum.each(fn %{device_id: device_id, pid: pid} ->
+      :telemetry.execute([:nerves_hub, :deployment, :trigger_update, :device], %{count: 1})
+
+      device = %Device{id: device_id}
+
+      # Check again because other nodes are processing at the same time
+      if Devices.count_inflight_updates_for(deployment) < deployment.concurrent_updates do
+        case Devices.told_to_update(device, deployment) do
+          {:ok, inflight_update} ->
+            send(pid, {"deployments/update", inflight_update})
+
+          :error ->
+            Logger.error(
+              "An inflight update could not be created or found for the device #{device.identifier} (#{device.id})"
+            )
         end
+      end
 
-        # Slow the update a bit to allow for concurrent nodes
-        Process.sleep(500)
-      end)
-    end
+      # Slow the update a bit to allow for concurrent nodes
+      Process.sleep(500)
+    end)
   end
 
   def init(deployment) do
diff --git a/mix.exs b/mix.exs
index f032ec4f1..11b6b6096 100644
--- a/mix.exs
+++ b/mix.exs
@@ -107,6 +107,7 @@ defmodule NervesHub.MixProject do
        branch: "update-obans-semantic-conventions",
        subdir: "instrumentation/opentelemetry_oban"},
       {:opentelemetry_bandit, "~> 0.2.0-rc.1"},
+      {:open_telemetry_decorator, "~> 1.5"},
       {:phoenix, "~> 1.7.0"},
       {:phoenix_ecto, "~> 4.0"},
       {:phoenix_html, "~> 3.3.1", override: true},
diff --git a/mix.lock b/mix.lock
index 45807f002..46ae067c6 100644
--- a/mix.lock
+++ b/mix.lock
@@ -52,7 +52,9 @@
   "nimble_ownership": {:hex, :nimble_ownership, "1.0.0", "3f87744d42c21b2042a0aa1d48c83c77e6dd9dd357e425a038dd4b49ba8b79a1", [:mix], [], "hexpm", "7c16cc74f4e952464220a73055b557a273e8b1b7ace8489ec9d86e9ad56cb2cc"},
   "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"},
   "nimble_strftime": {:hex, :nimble_strftime, "0.1.1", "b988184d1bd945bc139b2c27dd00a6c0774ec94f6b0b580083abd62d5d07818b", [:mix], [], "hexpm", "89e599c9b8b4d1203b7bb5c79eb51ef7c6a28fbc6228230b312f8b796310d755"},
+  "o11y": {:hex, :o11y, "0.2.6", "9386b36f9cc65ea838733b4838dea02334c4fb8721e53b904d02d50396df5cdc", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:opentelemetry_api, "~> 1.3", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}], "hexpm", "ea631c5ba03b155c1bc5849804da5d375538659446e4bb9388a0427959d122ef"},
   "oban": {:hex, :oban, "2.18.3", "1608c04f8856c108555c379f2f56bc0759149d35fa9d3b825cb8a6769f8ae926", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "36ca6ca84ef6518f9c2c759ea88efd438a3c81d667ba23b02b062a0aa785475e"},
+  "open_telemetry_decorator": {:hex, :open_telemetry_decorator, "1.5.7", "d0bb1942a3ea33e8fbf92ac44e084771a7dea076a559d974bf5b8ac11f76beaa", [:mix], [{:decorator, "~> 1.4", [hex: :decorator, repo: "hexpm", optional: false]}, {:o11y, "~> 0.2", [hex: :o11y, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.4", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}], "hexpm", "80566f68159c727c77986c617478a63c4e51256061baf52503d58d9e2316c6c4"},
   "opentelemetry": {:hex, :opentelemetry, "1.5.0", "7dda6551edfc3050ea4b0b40c0d2570423d6372b97e9c60793263ef62c53c3c2", [:rebar3], [{:opentelemetry_api, "~> 1.4", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}], "hexpm", "cdf4f51d17b592fc592b9a75f86a6f808c23044ba7cf7b9534debbcc5c23b0ee"},
   "opentelemetry_api": {:hex, :opentelemetry_api, "1.4.0", "63ca1742f92f00059298f478048dfb826f4b20d49534493d6919a0db39b6db04", [:mix, :rebar3], [], "hexpm", "3dfbbfaa2c2ed3121c5c483162836c4f9027def469c41578af5ef32589fcfc58"},
   "opentelemetry_bandit": {:hex, :opentelemetry_bandit, "0.2.0-rc.1", "3b78e4a472c20102af2d6852a04a890a8780de26feda560a63283045bc9088f5", [:mix], [{:nimble_options, "~> 1.1", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.3", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:opentelemetry_semantic_conventions, "~> 1.27", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: false]}, {:otel_http, "~> 0.2", [hex: :otel_http, repo: "hexpm", optional: false]}, {:plug, ">= 1.15.0", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.2", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9755c38e8b952204299f568cc30b5462851340c8b4b50702614ca403275b573a"},

From e05943a3bcd0f296878a10f53bbc6565014f43a2 Mon Sep 17 00:00:00 2001
From: Josh Kalderimis <josh.kalderimis@gmail.com>
Date: Sat, 26 Oct 2024 16:22:37 +1300
Subject: [PATCH 03/14] Add some initial tracing to the device socket and
 channel

---
 lib/nerves_hub_web/channels/device_channel.ex | 5 +++++
 lib/nerves_hub_web/channels/device_socket.ex  | 9 +++++++++
 2 files changed, 14 insertions(+)

diff --git a/lib/nerves_hub_web/channels/device_channel.ex b/lib/nerves_hub_web/channels/device_channel.ex
index 922f83078..9ba5cee34 100644
--- a/lib/nerves_hub_web/channels/device_channel.ex
+++ b/lib/nerves_hub_web/channels/device_channel.ex
@@ -6,6 +6,7 @@ defmodule NervesHubWeb.DeviceChannel do
   """
 
   use Phoenix.Channel
+  use OpenTelemetryDecorator
 
   require Logger
 
@@ -19,6 +20,7 @@ defmodule NervesHubWeb.DeviceChannel do
   alias NervesHub.Repo
   alias Phoenix.Socket.Broadcast
 
+  @decorate with_span("Channels.DeviceChannel.join")
   def join("device", params, %{assigns: %{device: device}} = socket) do
     with {:ok, device} <- update_metadata(device, params) do
       send(self(), {:after_join, params})
@@ -31,6 +33,7 @@ defmodule NervesHubWeb.DeviceChannel do
     end
   end
 
+  @decorate with_span("Channels.DeviceChannel.handle_info:after_join")
   def handle_info({:after_join, params}, %{assigns: %{device: device}} = socket) do
     device = maybe_update_deployment(device)
 
@@ -77,6 +80,7 @@ defmodule NervesHubWeb.DeviceChannel do
     {:stop, :shutdown, socket}
   end
 
+  @decorate with_span("Channels.DeviceChannel.handle_info:device_registration")
   def handle_info({:device_registation, attempt}, socket) do
     %{assigns: %{device: device}} = socket
 
@@ -215,6 +219,7 @@ defmodule NervesHubWeb.DeviceChannel do
   end
 
   # Update local state and tell the various servers of the new information
+  @decorate with_span("Channels.DeviceChannel.handle_info:devices-updated")
   def handle_info(%Broadcast{event: "devices/updated"}, %{assigns: %{device: device}} = socket) do
     device = Repo.reload(device)
 
diff --git a/lib/nerves_hub_web/channels/device_socket.ex b/lib/nerves_hub_web/channels/device_socket.ex
index c07a528f9..75fa02af6 100644
--- a/lib/nerves_hub_web/channels/device_socket.ex
+++ b/lib/nerves_hub_web/channels/device_socket.ex
@@ -1,5 +1,6 @@
 defmodule NervesHubWeb.DeviceSocket do
   use Phoenix.Socket
+  use OpenTelemetryDecorator
 
   require Logger
 
@@ -26,6 +27,7 @@ defmodule NervesHubWeb.DeviceSocket do
   end
 
   @impl Phoenix.Socket.Transport
+  @decorate with_span("Channels.DeviceSocket.terminate")
   def terminate(reason, {_channels_info, socket} = state) do
     on_disconnect(reason, socket)
     super(reason, state)
@@ -40,6 +42,7 @@ defmodule NervesHubWeb.DeviceSocket do
     super(msg, {state, socket})
   end
 
+  @decorate with_span("Channels.DeviceSocket.heartbeat")
   defp heartbeat(
          %Phoenix.Socket.Message{topic: "phoenix", event: "heartbeat"},
          %{assigns: %{device: device}} = socket
@@ -77,6 +80,7 @@ defmodule NervesHubWeb.DeviceSocket do
 
   # Used by Devices connecting with SSL certificates
   @impl Phoenix.Socket
+  @decorate with_span("Channels.DeviceSocket.connect")
   def connect(_params, socket, %{peer_data: %{ssl_cert: ssl_cert}})
       when not is_nil(ssl_cert) do
     X509.Certificate.from_der!(ssl_cert)
@@ -96,6 +100,7 @@ defmodule NervesHubWeb.DeviceSocket do
   end
 
   # Used by Devices connecting with HMAC Shared Secrets
+  @decorate with_span("Channels.DeviceSocket.connect")
   def connect(_params, socket, %{x_headers: x_headers})
       when is_list(x_headers) and length(x_headers) > 0 do
     headers = Map.new(x_headers)
@@ -208,6 +213,7 @@ defmodule NervesHubWeb.DeviceSocket do
     {:ok, socket}
   end
 
+  @decorate with_span("Channels.DeviceSocket.on_connect")
   defp on_connect(socket) do
     :telemetry.execute([:nerves_hub, :devices, :connect], %{count: 1}, %{
       ref_id: socket.assigns.reference_id,
@@ -223,6 +229,7 @@ defmodule NervesHubWeb.DeviceSocket do
     assign(socket, :device, device)
   end
 
+  @decorate with_span("Channels.DeviceSocket.on_disconnect")
   defp on_disconnect({:error, reason}, %{assigns: %{device: device, reference_id: reference_id}}) do
     if reason == {:shutdown, :disconnected} do
       :telemetry.execute([:nerves_hub, :devices, :duplicate_connection], %{count: 1}, %{
@@ -236,10 +243,12 @@ defmodule NervesHubWeb.DeviceSocket do
     :ok
   end
 
+  @decorate with_span("Channels.DeviceSocket.on_disconnect")
   defp on_disconnect(_, %{assigns: %{device: device, reference_id: reference_id}}) do
     shutdown(device, reference_id)
   end
 
+  @decorate with_span("Channels.DeviceSocket.shutdown")
   defp shutdown(device, reference_id) do
     :telemetry.execute([:nerves_hub, :devices, :disconnect], %{count: 1}, %{
       ref_id: reference_id,

From fa4d0f5ae23b9314d793bd72356c51ad7c5d24ce Mon Sep 17 00:00:00 2001
From: Josh Kalderimis <josh.kalderimis@gmail.com>
Date: Sat, 26 Oct 2024 19:30:32 +1300
Subject: [PATCH 04/14] More OTel decorators for Device related tracing

---
 lib/nerves_hub/deployments/orchestrator.ex    | 2 ++
 lib/nerves_hub_web/channels/device_channel.ex | 6 ++++++
 2 files changed, 8 insertions(+)

diff --git a/lib/nerves_hub/deployments/orchestrator.ex b/lib/nerves_hub/deployments/orchestrator.ex
index d3a2139ae..8e2fddbf6 100644
--- a/lib/nerves_hub/deployments/orchestrator.ex
+++ b/lib/nerves_hub/deployments/orchestrator.ex
@@ -108,6 +108,7 @@ defmodule NervesHub.Deployments.Orchestrator do
     {:ok, deployment, {:continue, :boot}}
   end
 
+  @decorate with_span("Deployments.Orchestrator.boot")
   def handle_continue(:boot, deployment) do
     _ = PubSub.subscribe(NervesHub.PubSub, "deployment:#{deployment.id}")
 
@@ -128,6 +129,7 @@ defmodule NervesHub.Deployments.Orchestrator do
     {:noreply, deployment}
   end
 
+  @decorate with_span("Deployments.Orchestrator.handle_info:deployments/update")
   def handle_info(%Broadcast{event: "deployments/update"}, deployment) do
     deployment =
       deployment
diff --git a/lib/nerves_hub_web/channels/device_channel.ex b/lib/nerves_hub_web/channels/device_channel.ex
index 9ba5cee34..49021e09d 100644
--- a/lib/nerves_hub_web/channels/device_channel.ex
+++ b/lib/nerves_hub_web/channels/device_channel.ex
@@ -102,6 +102,7 @@ defmodule NervesHubWeb.DeviceChannel do
 
   # We can save a fairly expensive query by checking the incoming deployment's payload
   # If it matches, we can set the deployment directly and only do 3 queries (update, two preloads)
+  @decorate with_span("Channels.DeviceChannel.handle_info:deployments/changed,deployment:none")
   def handle_info(
         %Broadcast{event: "deployments/changed", topic: "deployment:none", payload: payload},
         %{assigns: %{device: device}} = socket
@@ -125,6 +126,7 @@ defmodule NervesHubWeb.DeviceChannel do
     {:noreply, assign_deployment(socket, payload)}
   end
 
+  @decorate with_span("Channels.DeviceChannel.handle_info:deployments/changed")
   def handle_info(
         %Broadcast{event: "deployments/changed", payload: payload},
         %{assigns: %{device: device}} = socket
@@ -142,6 +144,7 @@ defmodule NervesHubWeb.DeviceChannel do
     end
   end
 
+  @decorate with_span("Channels.DeviceChannel.handle_info:resolve_changed_deployment")
   def handle_info(:resolve_changed_deployment, %{assigns: %{device: device}} = socket) do
     :telemetry.execute([:nerves_hub, :devices, :deployment, :changed], %{count: 1})
 
@@ -175,6 +178,7 @@ defmodule NervesHubWeb.DeviceChannel do
     {:noreply, socket}
   end
 
+  @decorate with_span("Channels.DeviceChannel.handle_info:deployments/update")
   def handle_info({"deployments/update", inflight_update}, %{assigns: %{device: device}} = socket) do
     device = deployment_preload(device)
 
@@ -360,6 +364,7 @@ defmodule NervesHubWeb.DeviceChannel do
     end
   end
 
+  @decorate with_span("Channels.DeviceChannel.handle_in:location:update")
   def handle_in("location:update", location, %{assigns: %{device: device}} = socket) do
     metadata = Map.put(device.connection_metadata, "location", location)
 
@@ -410,6 +415,7 @@ defmodule NervesHubWeb.DeviceChannel do
     {:noreply, socket}
   end
 
+  @decorate with_span("Channels.DeviceChannel.handle_in:health_check_report")
   def handle_in("health_check_report", %{"value" => device_status}, socket) do
     device_meta =
       for {key, val} <- Map.from_struct(socket.assigns.device.firmware_metadata),

From 4f318f2ef594f2817b057710a19446101025e319 Mon Sep 17 00:00:00 2001
From: Josh Kalderimis <josh.kalderimis@gmail.com>
Date: Sat, 26 Oct 2024 20:22:38 +1300
Subject: [PATCH 05/14] Disable tracing exporter during testing

---
 config/test.exs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/config/test.exs b/config/test.exs
index fe1bac8de..60807496a 100644
--- a/config/test.exs
+++ b/config/test.exs
@@ -91,3 +91,5 @@ config :phoenix_test, :endpoint, NervesHubWeb.Endpoint
 
 # Initialize plugs at runtime for faster test compilation
 config :phoenix, :plug_init_mode, :runtime
+
+config :opentelemetry, traces_exporter: :none

From 75dad41e2be9d24ee1378d2e2b82940cfe58c238 Mon Sep 17 00:00:00 2001
From: Josh Kalderimis <josh.kalderimis@gmail.com>
Date: Mon, 28 Oct 2024 20:04:14 +1300
Subject: [PATCH 06/14] Add a custom OTel filtered sampler

---
 config/dev.exs                               |  2 -
 config/runtime.exs                           | 27 +++---
 config/test.exs                              |  2 -
 lib/nerves_hub/application.ex                | 30 ++++---
 lib/nerves_hub/telemetry/filtered_sampler.ex | 88 ++++++++++++++++++++
 5 files changed, 119 insertions(+), 30 deletions(-)
 create mode 100644 lib/nerves_hub/telemetry/filtered_sampler.ex

diff --git a/config/dev.exs b/config/dev.exs
index 3e91c693c..90f729676 100644
--- a/config/dev.exs
+++ b/config/dev.exs
@@ -112,5 +112,3 @@ config :nerves_hub, NervesHub.RateLimit, limit: 10
 
 config :nerves_hub,
   open_for_registrations: true
-
-config :opentelemetry, traces_exporter: :none
diff --git a/config/runtime.exs b/config/runtime.exs
index e1c3e91e1..430f8280d 100644
--- a/config/runtime.exs
+++ b/config/runtime.exs
@@ -387,26 +387,25 @@ config :sentry,
     ]
   ]
 
-if otlp_endpoint = System.get_env("OTLP_ENDPOINT") do
-  config :opentelemetry, :resource, service: %{name: nerves_hub_app}
+config :opentelemetry, :resource, service: %{name: nerves_hub_app}
 
+if otlp_endpoint = System.get_env("OTLP_ENDPOINT") do
   config :opentelemetry_exporter,
     otlp_protocol: :http_protobuf,
     otlp_endpoint: otlp_endpoint,
     otlp_headers: [{System.get_env("OTLP_AUTH_HEADER"), System.get_env("OTLP_AUTH_HEADER_VALUE")}]
 
-  if otlp_sampler_ratio = System.get_env("OTLP_SAMPLER_RATIO") do
-    config :opentelemetry,
-      sampler:
-        {:parent_based,
-         %{
-           root: {:trace_id_ratio_based, String.to_float(otlp_sampler_ratio)},
-           remote_parent_sampled: :always_on,
-           remote_parent_not_sampled: :always_off,
-           local_parent_sampled: :always_on,
-           local_parent_not_sampled: :always_off
-         }}
-  end
+  otlp_sampler_ratio =
+    if ratio = System.get_env("OTLP_SAMPLER_RATIO") do
+      String.to_float(ratio)
+    else
+      nil
+    end
+
+  config :opentelemetry,
+    sampler: {:parent_based, %{root: {NervesHub.Telemetry.FilteredSampler, otlp_sampler_ratio}}}
+else
+  config :opentelemetry, traces_exporter: :none
 end
 
 if host = System.get_env("STATSD_HOST") do
diff --git a/config/test.exs b/config/test.exs
index 60807496a..fe1bac8de 100644
--- a/config/test.exs
+++ b/config/test.exs
@@ -91,5 +91,3 @@ config :phoenix_test, :endpoint, NervesHubWeb.Endpoint
 
 # Initialize plugs at runtime for faster test compilation
 config :phoenix, :plug_init_mode, :runtime
-
-config :opentelemetry, traces_exporter: :none
diff --git a/lib/nerves_hub/application.ex b/lib/nerves_hub/application.ex
index 0c9efb4ab..72a5284c5 100644
--- a/lib/nerves_hub/application.ex
+++ b/lib/nerves_hub/application.ex
@@ -12,18 +12,7 @@ defmodule NervesHub.Application do
         raise "fwup could not be found in the $PATH. This is a requirement of NervesHubWeb and cannot start otherwise"
     end
 
-    if System.get_env("ECTO_IPV6") do
-      :httpc.set_option(:ipfamily, :inet6fb4)
-    end
-
-    :ok = OpentelemetryBandit.setup()
-    :ok = OpentelemetryPhoenix.setup(adapter: :bandit)
-    :ok = OpentelemetryOban.setup(trace: [:jobs])
-
-    :ok =
-      NervesHub.Repo.config()
-      |> Keyword.fetch!(:telemetry_prefix)
-      |> OpentelemetryEcto.setup(db_statement: :enabled)
+    setup_open_telemetry()
 
     _ =
       :logger.add_handler(:my_sentry_handler, Sentry.LoggerHandler, %{
@@ -60,6 +49,23 @@ defmodule NervesHub.Application do
     Supervisor.start_link(children, opts)
   end
 
+  defp setup_open_telemetry() do
+    if System.get_env("ECTO_IPV6") do
+      :httpc.set_option(:ipfamily, :inet6fb4)
+    end
+
+    :ok = OpentelemetryBandit.setup()
+    :ok = OpentelemetryPhoenix.setup(adapter: :bandit)
+    :ok = OpentelemetryOban.setup(trace: [:jobs])
+
+    :ok =
+      NervesHub.Repo.config()
+      |> Keyword.fetch!(:telemetry_prefix)
+      |> OpentelemetryEcto.setup(db_statement: :enabled)
+
+    :ok
+  end
+
   def config_change(changed, _new, removed) do
     NervesHubWeb.Endpoint.config_change(changed, removed)
     :ok
diff --git a/lib/nerves_hub/telemetry/filtered_sampler.ex b/lib/nerves_hub/telemetry/filtered_sampler.ex
new file mode 100644
index 000000000..1de977f2c
--- /dev/null
+++ b/lib/nerves_hub/telemetry/filtered_sampler.ex
@@ -0,0 +1,88 @@
+defmodule NervesHub.Telemetry.FilteredSampler do
+  # Inspired by https://arathunku.com/b/2024/notes-on-adding-opentelemetry-to-an-elixir-app/
+
+  # TODO: Add ratio sampling support
+
+  require OpenTelemetry.Tracer, as: Tracer
+  require Logger
+
+  @behaviour :otel_sampler
+
+  @ignored_static_paths ~r/^\/(assets|fonts|images)\/.*/
+
+  @ignored_url_paths [
+    "/status/alive",
+    "/favicon.ico",
+    "/phoenix/live_reload/socket/websocket",
+    # "/live/websocket",
+    "/"
+  ]
+
+  @ignored_span_names [
+    "Channels.DeviceSocket.heartbeat",
+    "nerves_hub.repo.query:schema_migrations"
+  ]
+
+  @impl :otel_sampler
+  def setup(probability \\ nil) do
+    if probability do
+      [ratio_sampler_config: :otel_sampler_trace_id_ratio_based.setup(probability)]
+    else
+      []
+    end
+  end
+
+  @impl :otel_sampler
+  def description(_sampler_config), do: "NervesHub.Sampler"
+
+  @impl :otel_sampler
+  def should_sample(
+        ctx,
+        trace_id,
+        links,
+        span_name,
+        span_kind,
+        attributes,
+        sampler_config
+      ) do
+    result = drop_trace?(span_name, attributes)
+
+    tracestate = Tracer.current_span_ctx(ctx) |> OpenTelemetry.Span.tracestate()
+
+    case result do
+      true ->
+        {:drop, [], tracestate}
+
+      false ->
+        if config = sampler_config[:ratio_sampler_config] do
+          :otel_sampler_trace_id_ratio_based.should_sample(
+            ctx,
+            trace_id,
+            links,
+            span_name,
+            span_kind,
+            attributes,
+            config
+          )
+        else
+          {:record_and_sample, [], tracestate}
+        end
+    end
+  end
+
+  def drop_trace?(span_name, attributes) do
+    cond do
+      Enum.member?(@ignored_span_names, span_name) ->
+        true
+
+      span_name == :GET && Enum.member?(@ignored_url_paths, attributes[:"url.path"]) ->
+        true
+
+      span_name == :GET && (attributes[:"url.path"] || "") =~ @ignored_static_paths ->
+        true
+
+      true ->
+        false
+    end
+  end
+end

From 37d263afa5021ba5f18145b866357e791aad1bb8 Mon Sep 17 00:00:00 2001
From: Josh Kalderimis <josh.kalderimis@gmail.com>
Date: Mon, 28 Oct 2024 20:05:06 +1300
Subject: [PATCH 07/14] Update the span name for websocket requests

---
 lib/nerves_hub/application.ex              |  2 ++
 lib/nerves_hub/telemetry/customizations.ex | 23 ++++++++++++++++++++++
 2 files changed, 25 insertions(+)
 create mode 100644 lib/nerves_hub/telemetry/customizations.ex

diff --git a/lib/nerves_hub/application.ex b/lib/nerves_hub/application.ex
index 72a5284c5..088bea481 100644
--- a/lib/nerves_hub/application.ex
+++ b/lib/nerves_hub/application.ex
@@ -54,6 +54,8 @@ defmodule NervesHub.Application do
       :httpc.set_option(:ipfamily, :inet6fb4)
     end
 
+    :ok = NervesHub.Telemetry.Customizations.setup()
+
     :ok = OpentelemetryBandit.setup()
     :ok = OpentelemetryPhoenix.setup(adapter: :bandit)
     :ok = OpentelemetryOban.setup(trace: [:jobs])
diff --git a/lib/nerves_hub/telemetry/customizations.ex b/lib/nerves_hub/telemetry/customizations.ex
new file mode 100644
index 000000000..d8914e1af
--- /dev/null
+++ b/lib/nerves_hub/telemetry/customizations.ex
@@ -0,0 +1,23 @@
+defmodule NervesHub.Telemetry.Customizations do
+  alias OpenTelemetry.Tracer
+  require OpenTelemetry.Tracer
+
+  def setup() do
+    :telemetry.attach_many(
+      {__MODULE__, :bandit_customizations},
+      [
+        [:bandit, :request, :stop]
+      ],
+      &__MODULE__.handle_request/4,
+      nil
+    )
+  end
+
+  def handle_request([:bandit, :request, :stop], _measurements, %{conn: conn}, _config) do
+    if conn.status == 101 do
+      Tracer.update_name("WEBSOCKET #{conn.request_path}")
+    end
+
+    :ok
+  end
+end

From ee79faa2002c0e8cc0762d9cd806f6a3af4ef901 Mon Sep 17 00:00:00 2001
From: Josh Kalderimis <josh.kalderimis@gmail.com>
Date: Mon, 28 Oct 2024 22:16:18 +1300
Subject: [PATCH 08/14] Ignore the css static path

---
 lib/nerves_hub/telemetry/filtered_sampler.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/nerves_hub/telemetry/filtered_sampler.ex b/lib/nerves_hub/telemetry/filtered_sampler.ex
index 1de977f2c..5df9eb7e0 100644
--- a/lib/nerves_hub/telemetry/filtered_sampler.ex
+++ b/lib/nerves_hub/telemetry/filtered_sampler.ex
@@ -8,7 +8,7 @@ defmodule NervesHub.Telemetry.FilteredSampler do
 
   @behaviour :otel_sampler
 
-  @ignored_static_paths ~r/^\/(assets|fonts|images)\/.*/
+  @ignored_static_paths ~r/^\/(assets|fonts|images|css)\/.*/
 
   @ignored_url_paths [
     "/status/alive",

From 03524022278d50248ed0d538e0f0f23b41fe47b3 Mon Sep 17 00:00:00 2001
From: Josh Kalderimis <josh.kalderimis@gmail.com>
Date: Tue, 29 Oct 2024 09:05:17 +1300
Subject: [PATCH 09/14] Move around the ignored url paths

---
 lib/nerves_hub/telemetry/filtered_sampler.ex | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/nerves_hub/telemetry/filtered_sampler.ex b/lib/nerves_hub/telemetry/filtered_sampler.ex
index 5df9eb7e0..063f6fce2 100644
--- a/lib/nerves_hub/telemetry/filtered_sampler.ex
+++ b/lib/nerves_hub/telemetry/filtered_sampler.ex
@@ -12,9 +12,9 @@ defmodule NervesHub.Telemetry.FilteredSampler do
 
   @ignored_url_paths [
     "/status/alive",
-    "/favicon.ico",
     "/phoenix/live_reload/socket/websocket",
-    # "/live/websocket",
+    "/live/websocket",
+    "/favicon.ico",
     "/"
   ]
 

From 89a59ea9f6d4851a477ab9069ae5ea3da6f4f4f2 Mon Sep 17 00:00:00 2001
From: Josh Kalderimis <josh.kalderimis@gmail.com>
Date: Tue, 29 Oct 2024 09:05:53 +1300
Subject: [PATCH 10/14] Match on request path instead of status

---
 lib/nerves_hub/telemetry/customizations.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/nerves_hub/telemetry/customizations.ex b/lib/nerves_hub/telemetry/customizations.ex
index d8914e1af..528b231f4 100644
--- a/lib/nerves_hub/telemetry/customizations.ex
+++ b/lib/nerves_hub/telemetry/customizations.ex
@@ -14,7 +14,7 @@ defmodule NervesHub.Telemetry.Customizations do
   end
 
   def handle_request([:bandit, :request, :stop], _measurements, %{conn: conn}, _config) do
-    if conn.status == 101 do
+    if conn.request_path =~ ~r/\/websocket$/ do
       Tracer.update_name("WEBSOCKET #{conn.request_path}")
     end
 

From 875c6ccfbde3c6ef1d50824865dabb800767d5bc Mon Sep 17 00:00:00 2001
From: Josh Kalderimis <josh.kalderimis@gmail.com>
Date: Thu, 7 Nov 2024 13:12:46 +1300
Subject: [PATCH 11/14] `mix deps.get` after rebasing

---
 mix.lock | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/mix.lock b/mix.lock
index 46ae067c6..a9ad027f2 100644
--- a/mix.lock
+++ b/mix.lock
@@ -12,6 +12,7 @@
   "comeonin": {:hex, :comeonin, "5.5.0", "364d00df52545c44a139bad919d7eacb55abf39e86565878e17cebb787977368", [:mix], [], "hexpm", "6287fc3ba0aad34883cbe3f7949fc1d1e738e5ccdce77165bc99490aa69f47fb"},
   "contex": {:hex, :contex, "0.5.0", "5d8a6defbeb41f54adfcb0f85c4756d4f2b84aa5b0d809d45a5d2e90d91d0392", [:mix], [{:nimble_strftime, "~> 0.1.0", [hex: :nimble_strftime, repo: "hexpm", optional: false]}], "hexpm", "b7497a1790324d84247859df44ba4bcf2489d9bba1812a5375b2f2046b9e6fd7"},
   "crontab": {:hex, :crontab, "1.1.14", "233fcfdc2c74510cabdbcb800626babef414e7cb13cea11ddf62e10e16e2bf76", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm", "4e3b9950bc22ae8d0395ffb5f4b127a140005cba95745abf5ff9ee7e8203c6fa"},
+  "ctx": {:hex, :ctx, "0.6.0", "8ff88b70e6400c4df90142e7f130625b82086077a45364a78d208ed3ed53c7fe", [:rebar3], [], "hexpm", "a14ed2d1b67723dbebbe423b28d7615eb0bdcba6ff28f2d1f1b0a7e1d4aa5fc2"},
   "custom_base": {:hex, :custom_base, "0.2.1", "4a832a42ea0552299d81652aa0b1f775d462175293e99dfbe4d7dbaab785a706", [:mix], [], "hexpm", "8df019facc5ec9603e94f7270f1ac73ddf339f56ade76a721eaa57c1493ba463"},
   "db_connection": {:hex, :db_connection, "2.7.0", "b99faa9291bb09892c7da373bb82cba59aefa9b36300f6145c5f201c7adf48ec", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dcf08f31b2701f857dfc787fbad78223d61a32204f217f15e881dd93e4bdd3ff"},
   "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"},
@@ -31,6 +32,8 @@
   "floki": {:hex, :floki, "0.36.3", "1102f93b16a55bc5383b85ae3ec470f82dee056eaeff9195e8afdf0ef2a43c30", [:mix], [], "hexpm", "fe0158bff509e407735f6d40b3ee0d7deb47f3f3ee7c6c182ad28599f9f6b27a"},
   "gen_smtp": {:hex, :gen_smtp, "1.2.0", "9cfc75c72a8821588b9b9fe947ae5ab2aed95a052b81237e0928633a13276fd3", [:rebar3], [{:ranch, ">= 1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "5ee0375680bca8f20c4d85f58c2894441443a743355430ff33a783fe03296779"},
   "gettext": {:hex, :gettext, "0.26.2", "5978aa7b21fada6deabf1f6341ddba50bc69c999e812211903b169799208f2a8", [:mix], [{:expo, "~> 0.5.1 or ~> 1.0", [hex: :expo, repo: "hexpm", optional: false]}], "hexpm", "aa978504bcf76511efdc22d580ba08e2279caab1066b76bb9aa81c4a1e0a32a5"},
+  "gproc": {:hex, :gproc, "0.9.1", "f1df0364423539cf0b80e8201c8b1839e229e5f9b3ccb944c5834626998f5b8c", [:rebar3], [], "hexpm", "905088e32e72127ed9466f0bac0d8e65704ca5e73ee5a62cb073c3117916d507"},
+  "grpcbox": {:hex, :grpcbox, "0.17.1", "6e040ab3ef16fe699ffb513b0ef8e2e896da7b18931a1ef817143037c454bcce", [:rebar3], [{:acceptor_pool, "~> 1.0.0", [hex: :acceptor_pool, repo: "hexpm", optional: false]}, {:chatterbox, "~> 0.15.1", [hex: :ts_chatterbox, repo: "hexpm", optional: false]}, {:ctx, "~> 0.6.0", [hex: :ctx, repo: "hexpm", optional: false]}, {:gproc, "~> 0.9.1", [hex: :gproc, repo: "hexpm", optional: false]}], "hexpm", "4a3b5d7111daabc569dc9cbd9b202a3237d81c80bf97212fbc676832cb0ceb17"},
   "hackney": {:hex, :hackney, "1.20.1", "8d97aec62ddddd757d128bfd1df6c5861093419f8f7a4223823537bad5d064e2", [:rebar3], [{:certifi, "~> 2.12.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "fe9094e5f1a2a2c0a7d10918fee36bfec0ec2a979994cff8cfe8058cd9af38e3"},
   "hlclock": {:hex, :hlclock, "1.0.0", "7a72fc7a20a9382499216227edf97a8b118e21fc3fcad0e81b8d10c616ce1431", [:mix], [], "hexpm", "d3f994336a7fcbc68bf08b14b2101b61e57bef82c032a6e05c1cdc753612c941"},
   "hpack": {:hex, :hpack_erl, "0.3.0", "2461899cc4ab6a0ef8e970c1661c5fc6a52d3c25580bc6dd204f84ce94669926", [:rebar3], [], "hexpm", "d6137d7079169d8c485c6962dfe261af5b9ef60fbc557344511c1e65e3d95fb0"},

From 9698b1526a35d957ff93ce7249ce906475bd0236 Mon Sep 17 00:00:00 2001
From: Josh Kalderimis <josh.kalderimis@gmail.com>
Date: Thu, 7 Nov 2024 13:13:19 +1300
Subject: [PATCH 12/14] Add some extra span decorating

---
 lib/nerves_hub_web/channels/device_channel.ex | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/nerves_hub_web/channels/device_channel.ex b/lib/nerves_hub_web/channels/device_channel.ex
index 49021e09d..04b20004d 100644
--- a/lib/nerves_hub_web/channels/device_channel.ex
+++ b/lib/nerves_hub_web/channels/device_channel.ex
@@ -485,6 +485,7 @@ defmodule NervesHubWeb.DeviceChannel do
     :ok
   end
 
+  @decorate with_span("Channels.DeviceChannel.maybe_update_deployment")
   defp maybe_update_deployment(device) do
     device
     |> Deployments.preload_with_firmware_and_archive()

From 56164a72795b7ba08a6bc9172ba294b69b7092cf Mon Sep 17 00:00:00 2001
From: Josh Kalderimis <josh.kalderimis@gmail.com>
Date: Fri, 15 Nov 2024 17:01:33 +1300
Subject: [PATCH 13/14] Update `opentelemetry_oban` dep

---
 mix.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mix.lock b/mix.lock
index a9ad027f2..f07679da2 100644
--- a/mix.lock
+++ b/mix.lock
@@ -64,7 +64,7 @@
   "opentelemetry_ecto": {:hex, :opentelemetry_ecto, "1.2.0", "2382cb47ddc231f953d3b8263ed029d87fbf217915a1da82f49159d122b64865", [:mix], [{:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:opentelemetry_process_propagator, "~> 0.2", [hex: :opentelemetry_process_propagator, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "70dfa2e79932e86f209df00e36c980b17a32f82d175f0068bf7ef9a96cf080cf"},
   "opentelemetry_exporter": {:hex, :opentelemetry_exporter, "1.8.0", "5d546123230771ef4174e37bedfd77e3374913304cd6ea3ca82a2add49cd5d56", [:rebar3], [{:grpcbox, ">= 0.0.0", [hex: :grpcbox, repo: "hexpm", optional: false]}, {:opentelemetry, "~> 1.5.0", [hex: :opentelemetry, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.4.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:tls_certificate_check, "~> 1.18", [hex: :tls_certificate_check, repo: "hexpm", optional: false]}], "hexpm", "a1f9f271f8d3b02b81462a6bfef7075fd8457fdb06adff5d2537df5e2264d9af"},
   "opentelemetry_liveview": {:hex, :opentelemetry_liveview, "1.0.0-rc.4", "52915a83809100f31f7b6ea42e00b964a66032b75cc56e5b4cbcf7e21d4a45da", [:mix], [{:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:opentelemetry_telemetry, "~> 1.0.0-beta.7", [hex: :opentelemetry_telemetry, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e06ab69da7ee46158342cac42f1c22886bdeab53e8d8c4e237c3b3c2cf7b815d"},
-  "opentelemetry_oban": {:git, "https://github.com/joshk/opentelemetry-erlang-contrib", "f9b2a59870865fe7011edb65455b849e0c73547e", [branch: "update-obans-semantic-conventions", subdir: "instrumentation/opentelemetry_oban"]},
+  "opentelemetry_oban": {:git, "https://github.com/joshk/opentelemetry-erlang-contrib", "2ada0a1a7043706683701dfa1bda7560699be25b", [branch: "update-obans-semantic-conventions", subdir: "instrumentation/opentelemetry_oban"]},
   "opentelemetry_phoenix": {:hex, :opentelemetry_phoenix, "2.0.0-rc.1", "252f918a90f6aad4ea76133e03d9421b8014712d3d9186bd2bc54311563b282f", [:mix], [{:nimble_options, "~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.4", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:opentelemetry_process_propagator, "~> 0.3", [hex: :opentelemetry_process_propagator, repo: "hexpm", optional: false]}, {:opentelemetry_semantic_conventions, "~> 1.27", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: false]}, {:opentelemetry_telemetry, "~> 1.1", [hex: :opentelemetry_telemetry, repo: "hexpm", optional: false]}, {:otel_http, "~> 0.2", [hex: :otel_http, repo: "hexpm", optional: false]}, {:plug, ">= 1.11.0", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "74accdfd9f3757b802064f267ecbc5208a97fcb54d100e8e834b56fe09e64a8e"},
   "opentelemetry_process_propagator": {:hex, :opentelemetry_process_propagator, "0.3.0", "ef5b2059403a1e2b2d2c65914e6962e56371570b8c3ab5323d7a8d3444fb7f84", [:mix, :rebar3], [{:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}], "hexpm", "7243cb6de1523c473cba5b1aefa3f85e1ff8cc75d08f367104c1e11919c8c029"},
   "opentelemetry_semantic_conventions": {:hex, :opentelemetry_semantic_conventions, "1.27.0", "acd0194a94a1e57d63da982ee9f4a9f88834ae0b31b0bd850815fe9be4bbb45f", [:mix, :rebar3], [], "hexpm", "9681ccaa24fd3d810b4461581717661fd85ff7019b082c2dff89c7d5b1fc2864"},

From 5ca35338615166fa9e1b04507896622a960ac117 Mon Sep 17 00:00:00 2001
From: Josh Kalderimis <josh.kalderimis@gmail.com>
Date: Mon, 18 Nov 2024 11:35:16 +1300
Subject: [PATCH 14/14] Formatting fun times

---
 lib/nerves_hub_web/channels/device_socket.ex | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/nerves_hub_web/channels/device_socket.ex b/lib/nerves_hub_web/channels/device_socket.ex
index 2792d4263..3293c042d 100644
--- a/lib/nerves_hub_web/channels/device_socket.ex
+++ b/lib/nerves_hub_web/channels/device_socket.ex
@@ -243,8 +243,8 @@ defmodule NervesHubWeb.DeviceSocket do
   end
 
   @decorate with_span("Channels.DeviceSocket.on_disconnect")
-  defp on_disconnect(exit_reason, socket) 
-    
+  defp on_disconnect(exit_reason, socket)
+
   defp on_disconnect({:error, reason}, %{
          assigns: %{
            device: device,