Skip to content

Commit a362ae1

Browse files
joshkjoshk
committed
Add support for OpenTelemetry tracing
1 parent 8dc935b commit a362ae1

File tree

7 files changed

+128
-54
lines changed

7 files changed

+128
-54
lines changed

‎config/dev.exs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,3 +112,5 @@ config :nerves_hub, NervesHub.RateLimit, limit: 10
112112

113113
config :nerves_hub,
114114
open_for_registrations: true
115+
116+
config :opentelemetry, traces_exporter: :none

‎config/runtime.exs‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -387,6 +387,28 @@ config :sentry,
387387
]
388388
]
389389

390+
if otlp_endpoint = System.get_env("OTLP_ENDPOINT") do
391+
config :opentelemetry, :resource, service: %{name: nerves_hub_app}
392+
393+
config :opentelemetry_exporter,
394+
otlp_protocol: :http_protobuf,
395+
otlp_endpoint: otlp_endpoint,
396+
otlp_headers: [{System.get_env("OTLP_AUTH_HEADER"), System.get_env("OTLP_AUTH_HEADER_VALUE")}]
397+
398+
if otlp_sampler_ratio = System.get_env("OTLP_SAMPLER_RATIO") do
399+
config :opentelemetry,
400+
sampler:
401+
{:parent_based,
402+
%{
403+
root: {:trace_id_ratio_based, String.to_float(otlp_sampler_ratio)},
404+
remote_parent_sampled: :always_on,
405+
remote_parent_not_sampled: :always_off,
406+
local_parent_sampled: :always_on,
407+
local_parent_not_sampled: :always_off
408+
}}
409+
end
410+
end
411+
390412
if host = System.get_env("STATSD_HOST") do
391413
config :nerves_hub, :statsd,
392414
host: System.get_env("STATSD_HOST"),

‎lib/nerves_hub/application.ex‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,19 @@ defmodule NervesHub.Application do
1212
raise "fwup could not be found in the $PATH. This is a requirement of NervesHubWeb and cannot start otherwise"
1313
end
1414

15+
if System.get_env("ECTO_IPV6") do
16+
:httpc.set_option(:ipfamily, :inet6fb4)
17+
end
18+
19+
:ok = OpentelemetryBandit.setup()
20+
:ok = OpentelemetryPhoenix.setup(adapter: :bandit)
21+
:ok = OpentelemetryOban.setup(trace: [:jobs])
22+
23+
:ok =
24+
NervesHub.Repo.config()
25+
|> Keyword.fetch!(:telemetry_prefix)
26+
|> OpentelemetryEcto.setup(db_statement: :enabled)
27+
1528
_ =
1629
:logger.add_handler(:my_sentry_handler, Sentry.LoggerHandler, %{
1730
config: %{metadata: [:file, :line]}

‎lib/nerves_hub/deployments/orchestrator.ex‎

Lines changed: 53 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ defmodule NervesHub.Deployments.Orchestrator do
1212

1313
require Logger
1414

15+
require OpenTelemetry.Tracer, as: Tracer
16+
1517
alias NervesHub.Devices
1618
alias NervesHub.Devices.Device
1719
alias NervesHub.Repo
@@ -49,57 +51,59 @@ defmodule NervesHub.Deployments.Orchestrator do
4951
was successful, and the process is repeated.
5052
"""
5153
def trigger_update(deployment) do
52-
:telemetry.execute([:nerves_hub, :deployment, :trigger_update], %{count: 1})
53-
54-
match_conditions = [
55-
{:and, {:==, {:map_get, :deployment_id, :"$1"}, deployment.id},
56-
{:==, {:map_get, :updating, :"$1"}, false},
57-
{:==, {:map_get, :updates_enabled, :"$1"}, true},
58-
{:"/=", {:map_get, :firmware_uuid, :"$1"}, deployment.firmware.uuid}}
59-
]
60-
61-
match_return = %{
62-
device_id: {:element, 1, :"$_"},
63-
pid: {:element, 1, {:element, 2, :"$_"}},
64-
firmware_uuid: {:map_get, :firmware_uuid, {:element, 2, {:element, 2, :"$_"}}}
65-
}
66-
67-
devices =
68-
Registry.select(NervesHub.Devices.Registry, [
69-
{{:_, :_, :"$1"}, match_conditions, [match_return]}
70-
])
71-
72-
# Get a rough count of devices to update
73-
count = deployment.concurrent_updates - Devices.count_inflight_updates_for(deployment)
74-
# Just in case inflight goes higher than concurrent, limit it to 0
75-
count = max(count, 0)
76-
77-
# use a reduce to bounce out early?
78-
# limit the number of devices to 5 minutes / 500ms?
79-
80-
devices
81-
|> Enum.take(count)
82-
|> Enum.each(fn %{device_id: device_id, pid: pid} ->
83-
:telemetry.execute([:nerves_hub, :deployment, :trigger_update, :device], %{count: 1})
84-
85-
device = %Device{id: device_id}
86-
87-
# Check again because other nodes are processing at the same time
88-
if Devices.count_inflight_updates_for(deployment) < deployment.concurrent_updates do
89-
case Devices.told_to_update(device, deployment) do
90-
{:ok, inflight_update} ->
91-
send(pid, {"deployments/update", inflight_update})
92-
93-
:error ->
94-
Logger.error(
95-
"An inflight update could not be created or found for the device #{device.identifier} (#{device.id})"
96-
)
54+
Tracer.with_span "NervesHub.Deployments.Orchestrator.trigger_update" do
55+
:telemetry.execute([:nerves_hub, :deployment, :trigger_update], %{count: 1})
56+
57+
match_conditions = [
58+
{:and, {:==, {:map_get, :deployment_id, :"$1"}, deployment.id},
59+
{:==, {:map_get, :updating, :"$1"}, false},
60+
{:==, {:map_get, :updates_enabled, :"$1"}, true},
61+
{:"/=", {:map_get, :firmware_uuid, :"$1"}, deployment.firmware.uuid}}
62+
]
63+
64+
match_return = %{
65+
device_id: {:element, 1, :"$_"},
66+
pid: {:element, 1, {:element, 2, :"$_"}},
67+
firmware_uuid: {:map_get, :firmware_uuid, {:element, 2, {:element, 2, :"$_"}}}
68+
}
69+
70+
devices =
71+
Registry.select(NervesHub.Devices.Registry, [
72+
{{:_, :_, :"$1"}, match_conditions, [match_return]}
73+
])
74+
75+
# Get a rough count of devices to update
76+
count = deployment.concurrent_updates - Devices.count_inflight_updates_for(deployment)
77+
# Just in case inflight goes higher than concurrent, limit it to 0
78+
count = max(count, 0)
79+
80+
# use a reduce to bounce out early?
81+
# limit the number of devices to 5 minutes / 500ms?
82+
83+
devices
84+
|> Enum.take(count)
85+
|> Enum.each(fn %{device_id: device_id, pid: pid} ->
86+
:telemetry.execute([:nerves_hub, :deployment, :trigger_update, :device], %{count: 1})
87+
88+
device = %Device{id: device_id}
89+
90+
# Check again because other nodes are processing at the same time
91+
if Devices.count_inflight_updates_for(deployment) < deployment.concurrent_updates do
92+
case Devices.told_to_update(device, deployment) do
93+
{:ok, inflight_update} ->
94+
send(pid, {"deployments/update", inflight_update})
95+
96+
:error ->
97+
Logger.error(
98+
"An inflight update could not be created or found for the device #{device.identifier} (#{device.id})"
99+
)
100+
end
97101
end
98-
end
99102

100-
# Slow the update a bit to allow for concurrent nodes
101-
Process.sleep(500)
102-
end)
103+
# Slow the update a bit to allow for concurrent nodes
104+
Process.sleep(500)
105+
end)
106+
end
103107
end
104108

105109
def init(deployment) do

‎lib/nerves_hub_web/live/devices/index.ex‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ defmodule NervesHubWeb.Live.Devices.Index do
33

44
require Logger
55

6+
require OpenTelemetry.Tracer, as: Tracer
7+
68
alias NervesHub.AuditLogs
79
alias NervesHub.Devices
810
alias NervesHub.Devices.Alarms
@@ -356,12 +358,14 @@ defmodule NervesHubWeb.Live.Devices.Index do
356358
end
357359

358360
def handle_info(:refresh_device_list, socket) do
359-
Process.send_after(self(), :refresh_device_list, @list_refresh_time)
361+
Tracer.with_span "NervesHubWeb.Live.Devices.Index.refresh_device_list" do
362+
Process.send_after(self(), :refresh_device_list, @list_refresh_time)
360363

361-
if socket.assigns.paginate_opts.total_pages == 1 do
362-
{:noreply, assign_display_devices(socket)}
363-
else
364-
{:noreply, socket}
364+
if socket.assigns.paginate_opts.total_pages == 1 do
365+
{:noreply, assign_display_devices(socket)}
366+
else
367+
{:noreply, socket}
368+
end
365369
end
366370
end
367371

‎mix.exs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ defmodule NervesHub.MixProject do
1818
steps: [:assemble],
1919
include_executables_for: [:unix],
2020
applications: [
21+
opentelemetry_exporter: :permanent,
22+
opentelemetry: :temporary,
2123
nerves_hub: :permanent
2224
]
2325
]
@@ -95,6 +97,16 @@ defmodule NervesHub.MixProject do
9597
{:mox, "~> 1.0", only: [:test, :dev]},
9698
{:nimble_csv, "~> 1.1"},
9799
{:oban, "~> 2.11"},
100+
{:opentelemetry_exporter, "~> 1.8"},
101+
{:opentelemetry, "~> 1.5"},
102+
{:opentelemetry_api, "~> 1.4"},
103+
{:opentelemetry_ecto, "~> 1.2"},
104+
{:opentelemetry_phoenix, "~> 2.0.0-rc.1 "},
105+
{:opentelemetry_oban, "~> 1.0",
106+
git: "https://github.com/joshk/opentelemetry-erlang-contrib",
107+
branch: "update-obans-semantic-conventions",
108+
subdir: "instrumentation/opentelemetry_oban"},
109+
{:opentelemetry_bandit, "~> 0.2.0-rc.1"},
98110
{:phoenix, "~> 1.7.0"},
99111
{:phoenix_ecto, "~> 4.0"},
100112
{:phoenix_html, "~> 3.3.1", override: true},

0 commit comments

Comments
 (0)