Support Grafana via prom_ex

config/config.exs

@@ -48,6 +48,11 @@ config :nerves_hub, NervesHubWeb.Endpoint,
   ],
   pubsub_server: NervesHub.PubSub
 
+##
+# NervesHub Metrics
+#
+config :nerves_hub, NervesHubWeb.MetricsEndpoint, adapter: Bandit.PhoenixAdapter
+
 ##
 # Database and Oban
 #

config/dev.exs

@@ -66,6 +66,11 @@ config :nerves_hub, NervesHubWeb.DeviceEndpoint,
     ]
   ]
 
+##
+# NervesHub Metrics
+#
+config :nerves_hub, NervesHubWeb.MetricsEndpoint, http: [ip: {0, 0, 0, 0}, port: 4002]
+
 ##
 # Database and Oban
 #

config/prod.exs

@@ -9,13 +9,20 @@ config :phoenix, logger: false
 # NervesHub Web
 #
 config :nerves_hub, NervesHubWeb.Endpoint,
-  server: true,
-  force_ssl: [rewrite_on: [:x_forwarded_proto]]
+  force_ssl: [rewrite_on: [:x_forwarded_proto]],
+  server: true
 
 ##
 # NervesHub Device
 #
-config :nerves_hub, NervesHubWeb.DeviceEndpoint, server: true
+config :nerves_hub, NervesHubWeb.DeviceEndpoint,
+  force_ssl: [rewrite_on: [:x_forwarded_proto]],
+  server: true
+
+##
+# NervesHub Metrics
+#
+config :nerves_hub, NervesHubWeb.MetricsEndpoint, server: true
 
 ##
 # Database and Oban

config/runtime.exs

@@ -168,6 +168,18 @@ if config_env() == :prod do
       ]
   end
 
+  web_port = System.get_env("HTTP_PORT") || System.get_env("PORT") || "4000"
+
+  metrics_port =
+    case System.get_env("METRICS_PORT") do
+      nil -> String.to_integer(web_port) + 40
+      port -> String.to_integer(port)
+    end
+
+  config :nerves_hub, NervesHubWeb.MetricsEndpoint,
+    http: [port: metrics_port],
+    server: true
+
   config :nerves_hub, NervesHubWeb.DeviceSocket,
     shared_secrets: [
       enabled: System.get_env("DEVICE_SHARED_SECRETS_ENABLED", "false") == "true"
@@ -358,6 +370,18 @@ if System.get_env("SENTRY_DSN_URL") do
     before_send: {NervesHubWeb.SentryEventFilter, :filter_non_500}
 end
 
+if System.get_env("GRAFANA_HOST") do
+  config :nerves_hub, NervesHub.PromEx,
+    manual_metrics_start_delay: :no_delay,
+    grafana: [
+      host: System.get_env("GRAFANA_HOST"),
+      auth_token: System.get_env("GRAFANA_TOKEN") || raise("GRAFANA_TOKEN is required"),
+      upload_dashboards_on_start: true,
+      folder_name: "NervesHub App Dashboards",
+      annotate_app_lifecycle: true
+    ]
+end
+
 config :nerves_hub, :statsd,
   host: System.get_env("STATSD_HOST", "localhost"),
   port: String.to_integer(System.get_env("STATSD_PORT", "8125"))

lib/nerves_hub/application.ex

@@ -31,6 +31,8 @@ defmodule NervesHub.Application do
         [
           {Cachex, name: :geo_ip},
           NervesHub.RateLimit,
+          # PromEx needs to be started before Repo and Endpoint so it can capture some of their early events
+          NervesHub.PromEx,
           NervesHub.Repo,
           NervesHub.ObanRepo,
           {Phoenix.PubSub, name: NervesHub.PubSub},
@@ -39,7 +41,10 @@ defmodule NervesHub.Application do
           {Oban, Application.fetch_env!(:nerves_hub, Oban)}
         ] ++
         deployments_supervisor(deploy_env()) ++
-        endpoints(deploy_env())
+        endpoints(deploy_env()) ++
+        [
+          NervesHubWeb.MetricsEndpoint
+        ]
 
     opts = [strategy: :one_for_one, name: NervesHub.Supervisor]
     Supervisor.start_link(children, opts)

lib/nerves_hub/prom_ex.ex

@@ -0,0 +1,44 @@
+defmodule NervesHub.PromEx do
+  use PromEx, otp_app: :nerves_hub
+
+  alias PromEx.Plugins
+
+  @impl true
+  def plugins do
+    [
+      # PromEx built in plugins
+      Plugins.Application,
+      Plugins.Beam,
+      {
+        PromEx.Plugins.Phoenix,
+        endpoints: [
+          {NervesHubWeb.Endpoint, routers: [NervesHubWeb.Router]},
+          # DeviceEndpoint doesn't use a Router, but we need to pass to PromEx
+          {NervesHubWeb.DeviceEndpoint, routers: [NervesHubWeb.Router]}
+        ]
+      },
+      Plugins.PhoenixLiveView,
+      Plugins.Ecto,
+      Plugins.Oban
+    ]
+  end
+
+  @impl true
+  def dashboard_assigns do
+    [
+      datasource_id: "prometheus"
+    ]
+  end
+
+  @impl true
+  def dashboards do
+    [
+      # PromEx built in Grafana dashboards
+      {:prom_ex, "beam.json"},
+      {:nerves_hub, "grafana/phoenix.json"},
+      {:nerves_hub, "grafana/phoenix_live_view.json"},
+      {:nerves_hub, "grafana/ecto.json"},
+      {:nerves_hub, "grafana/oban.json"}
+    ]
+  end
+end

lib/nerves_hub_web/endpoint.ex

@@ -60,12 +60,13 @@ defmodule NervesHubWeb.Endpoint do
     cookie_key: "request_logger"
   )
 
+  # We don't need to see these in the logs, thus this is defined before telemetry
+  plug(NervesHubWeb.Plugs.ImAlive)
+
   plug(Plug.RequestId)
   plug(Plug.Telemetry, event_prefix: [:phoenix, :endpoint])
   plug(NervesHubWeb.Plugs.Logger)
 
-  plug(NervesHubWeb.Plugs.ImAlive)
-
   plug(
     Plug.Parsers,
     parsers: [:urlencoded, :multipart, :json],

lib/nerves_hub_web/metrics_endpoint.ex

@@ -0,0 +1,7 @@
+defmodule NervesHubWeb.MetricsEndpoint do
+  use Phoenix.Endpoint, otp_app: :nerves_hub
+
+  plug(PromEx.Plug, prom_ex_module: NervesHub.PromEx)
+
+  plug(NervesHubWeb.Plugs.DeviceEndpointRedirect)
+end

lib/nerves_hub_web/plugs/api/logger.ex

@@ -18,7 +18,7 @@ defmodule NervesHubWeb.API.Plugs.Logger do
           method: conn.method,
           path: request_path(conn),
           status: conn.status,
-          remote_ip: formatted_ip(conn.remote_ip)
+          remote_ip: formatted_ip(conn)
         )
       end)
 
@@ -33,7 +33,18 @@ defmodule NervesHubWeb.API.Plugs.Logger do
   def request_path(%{request_path: request_path}), do: request_path
   def request_path(_), do: nil
 
-  defp formatted_ip(ip) do
-    to_string(:inet_parse.ntoa(ip))
+  defp formatted_ip(conn) do
+    case Plug.Conn.get_req_header(conn, "x-forwarded-for") do
+      [ips] ->
+        ips
+        |> String.split(",")
+        |> List.first()
+        |> String.trim()
+
+      _ ->
+        conn.remote_ip
+        |> :inet_parse.ntoa()
+        |> to_string()
+    end
   end
 end

lib/nerves_hub_web/plugs/logger.ex

@@ -18,7 +18,7 @@ defmodule NervesHubWeb.Plugs.Logger do
           method: conn.method,
           path: request_path(conn),
           status: conn.status,
-          remote_ip: formatted_ip(conn.remote_ip)
+          remote_ip: formatted_ip(conn)
         )
       end)
 
@@ -33,7 +33,18 @@ defmodule NervesHubWeb.Plugs.Logger do
   def request_path(%{request_path: request_path}), do: request_path
   def request_path(_), do: nil
 
-  defp formatted_ip(ip) do
-    to_string(:inet_parse.ntoa(ip))
+  defp formatted_ip(conn) do
+    case Plug.Conn.get_req_header(conn, "x-forwarded-for") do
+      [ips] ->
+        ips
+        |> String.split(",")
+        |> List.first()
+        |> String.trim()
+
+      _ ->
+        conn.remote_ip
+        |> :inet_parse.ntoa()
+        |> to_string()
+    end
   end
 end

mix.exs

@@ -95,16 +95,17 @@ defmodule NervesHub.MixProject do
       {:phoenix_pubsub, "~> 2.0"},
       {:phoenix_swoosh, "~> 1.0"},
       {:phoenix_view, "~> 2.0"},
-      {:phoenix_test, "~> 0.3.0", only: :test, runtime: false},
+      {:phoenix_test, "0.3.1", only: :test, runtime: false},
       {:plug, "~> 1.7"},
       {:postgrex, "~> 0.14"},
+      {:prom_ex, "~> 1.10", git: "https://github.com/joshk/prom_ex", ref: "updated-with-prs"},
       {:scrivener_ecto, "~> 2.7"},
       {:scrivener_html, git: "https://github.com/nerves-hub/scrivener_html", branch: "phx-1.5"},
       {:sentry, "~> 10.0"},
       {:slipstream, "~> 1.0", only: [:test, :dev]},
       {:sweet_xml, "~> 0.6"},
       {:swoosh, "~> 1.12"},
-      {:telemetry_metrics, "~> 0.4"},
+      {:telemetry_metrics, "~> 1.0"},
       {:telemetry_metrics_statsd, "~> 0.7.0"},
       {:telemetry_poller, "~> 1.0"},
       {:timex, "~> 3.1"},