Very basic health status report for device (#1805)

A very very simple module for getting device health status based on
latest metric set, together with tests.

Not sure if it's even worth merging? Might just be a start for further
work on health statuses.

Note: default thresholds are currently just placeholders and should
probably be updated.

---------

Co-authored-by: Josh Kalderimis <[email protected]>

Author: elinol

assets/package-lock.json

@@ -11,6 +11,7 @@
     "check_formatting": "prettier -c js/**/*.js"
   },
   "dependencies": {
+    "@floating-ui/dom": "^1.6.13",
     "@mapbox/mapbox-gl-geocoder": "^5.0.3",
     "bootstrap": "^4.6.0",
     "chart.js": "^4.4.4",

assets/package.json

@@ -44,10 +44,18 @@
     @apply pl-6;
   }
 
+  .listing tr th:nth-child(2) {
+    @apply px-0;
+  }
+
   .listing tr td:first-child {
     @apply pl-6;
   }
 
+  .listing tr td:nth-child(2) {
+    @apply px-0;
+  }
+
   .listing .checkbox {
     @apply h-11 w-16 py-3 px-6;
   }

assets/ui-rework/app.css

@@ -14,6 +14,7 @@ import Chart from "./hooks/chart.js"
 import HighlightCode from "./hooks/highlightCode.js"
 import UpdatingTimeAgo from "./hooks/updatingTimeAgo.js"
 import LocalTime from "./hooks/localTime.js"
+import ToolTip from "./hooks/toolTip.js"
 import SimpleDate from "./hooks/simpleDate.js"
 import WorldMap from "./hooks/worldMap.js"
 import DeviceLocationMap from "./hooks/deviceLocationMap.js"
@@ -35,6 +36,7 @@ let liveSocket = new LiveSocket("/live", Socket, {
     HighlightCode,
     UpdatingTimeAgo,
     LocalTime,
+    ToolTip,
     SimpleDate,
     WorldMap,
     DeviceLocationMap,

assets/ui-rework/app.js

@@ -0,0 +1,46 @@
+import { computePosition, offset, arrow } from "@floating-ui/dom"
+
+export default {
+  mounted() {
+    this.updated()
+  },
+
+  updateTooltip() {
+    computePosition(this.el, this.content, {
+      placement: this.placement,
+      middleware: [offset(15), arrow({ element: this.arrow })]
+    }).then(({ x, y, middlewareData }) => {
+      Object.assign(this.content.style, {
+        top: `${y}px`,
+        left: `${x}px`
+      })
+
+      if (middlewareData.arrow) {
+        const { x } = middlewareData.arrow
+
+        Object.assign(this.arrow.style, {
+          left: `${x}px`,
+          top: `${-this.arrow.offsetHeight / 2}px`
+        })
+      }
+    })
+  },
+
+  showTooltip() {
+    this.content.style.display = "block"
+    this.updateTooltip()
+  },
+
+  hideTooltip() {
+    this.content.style.display = ""
+  },
+
+  updated() {
+    this.content = this.el.getElementsByClassName("tooltip-content")[0]
+    this.arrow = this.el.getElementsByClassName("tooltip-arrow")[0]
+    this.placement = this.el.dataset.placement || "bottom"
+
+    this.el.addEventListener("mouseover", this.showTooltip.bind(this))
+    this.el.addEventListener("mouseout", this.hideTooltip.bind(this))
+  }
+}

assets/ui-rework/hooks/toolTip.js

@@ -23,6 +23,8 @@ config :nerves_hub,
     System.get_env("DEVICE_ENDPOINT_REDIRECT", "https://docs.nerves-hub.org/"),
   device_health_days_to_retain:
     String.to_integer(System.get_env("HEALTH_CHECK_DAYS_TO_RETAIN", "7")),
+  device_health_delete_limit:
+    String.to_integer(System.get_env("DEVICE_HEALTH_DELETE_LIMIT", "100000")),
   device_deployment_change_jitter_seconds:
     String.to_integer(System.get_env("DEVICE_DEPLOYMENT_CHANGE_JITTER_SECONDS", "10")),
   device_last_seen_update_interval_minutes:

config/runtime.exs

@@ -77,14 +77,16 @@ defmodule NervesHub.Devices do
     |> join(:left, [d, o, p], dp in assoc(d, :deployment))
     |> join(:left, [d, o, p, dp], f in assoc(dp, :firmware))
     |> join(:left, [d, o, p, dp, f], lc in assoc(d, :latest_connection), as: :latest_connection)
+    |> join(:left, [d, o, p, dp, f, lc], lh in assoc(d, :latest_health), as: :latest_health)
     |> Repo.exclude_deleted()
     |> sort_devices(sorting)
     |> Filtering.build_filters(filters)
-    |> preload([d, o, p, dp, f, latest_connection: lc],
+    |> preload([d, o, p, dp, f, latest_connection: lc, latest_health: lh],
       org: o,
       product: p,
       deployment: {dp, firmware: f},
-      latest_connection: lc
+      latest_connection: lc,
+      latest_health: lh
     )
     |> Flop.run(flop)
   end
@@ -120,7 +122,9 @@ defmodule NervesHub.Devices do
     |> where([d], d.product_id == ^product.id)
     |> Repo.exclude_deleted()
     |> join(:left, [d], dc in assoc(d, :latest_connection), as: :latest_connection)
+    |> join(:left, [d, dc], dh in assoc(d, :latest_health), as: :latest_health)
     |> preload([latest_connection: lc], latest_connection: lc)
+    |> preload([latest_health: lh], latest_health: lh)
     |> Filtering.build_filters(filters)
     |> sort_devices(sorting)
     |> Flop.run(flop)
@@ -243,6 +247,12 @@ defmodule NervesHub.Devices do
     |> preload([d, o, dp], org: o, deployment: dp)
   end
 
+  defp join_and_preload(query, assocs) when is_list(assocs) do
+    Enum.reduce(assocs, query, fn assoc, q ->
+      join_and_preload(q, assoc)
+    end)
+  end
+
   defp join_and_preload(query, nil), do: query
 
   defp join_and_preload(query, :device_certificates) do
@@ -257,6 +267,12 @@ defmodule NervesHub.Devices do
     |> preload([latest_connection: lc], latest_connection: lc)
   end
 
+  defp join_and_preload(query, :latest_health) do
+    query
+    |> join(:left, [d], dh in assoc(d, :latest_health), as: :latest_health)
+    |> preload([latest_health: lh], latest_health: lh)
+  end
+
   def get_device_by_x509(cert) do
     fingerprint = NervesHub.Certificate.fingerprint(cert)
 
@@ -1145,23 +1161,52 @@ defmodule NervesHub.Devices do
   end
 
   def save_device_health(device_status) do
-    device_status
-    |> DeviceHealth.save()
-    |> Repo.insert()
+    Multi.new()
+    |> Multi.insert(:insert_health, DeviceHealth.save(device_status))
+    |> Ecto.Multi.update_all(:update_device, &update_health_on_device/1, [])
+    |> Repo.transaction()
+    |> case do
+      {:ok, %{insert_health: health}} ->
+        {:ok, health}
+
+      {:error, _, changeset, _} ->
+        {:error, changeset}
+    end
+  end
+
+  defp update_health_on_device(%{insert_health: health}) do
+    Device
+    |> where(id: ^health.device_id)
+    |> update(set: [latest_health_id: ^health.id])
   end
 
   def truncate_device_health() do
-    days_to_retain =
+    interval =
       Application.get_env(:nerves_hub, :device_health_days_to_retain)
 
-    days_ago = DateTime.shift(DateTime.utc_now(), day: -days_to_retain)
+    delete_limit = Application.get_env(:nerves_hub, :device_health_delete_limit)
+    time_ago = DateTime.shift(DateTime.utc_now(), day: -interval)
 
-    {count, _} =
+    query =
       DeviceHealth
-      |> where([dh], dh.inserted_at < ^days_ago)
-      |> Repo.delete_all()
+      |> join(:inner, [dh], d in Device, on: dh.device_id == d.id)
+      |> where([dh, _d], dh.inserted_at < ^time_ago)
+      |> where([dh, d], dh.id != d.latest_health_id)
+      |> select([dh], dh.id)
+      |> limit(^delete_limit)
 
-    {:ok, count}
+    {delete_count, _} =
+      DeviceHealth
+      |> where([dh], dh.id in subquery(query))
+      |> Repo.delete_all(timeout: 30_000)
+
+    if delete_count == 0 do
+      :ok
+    else
+      # relax stress on Ecto pool and go again
+      Process.sleep(2000)
+      truncate_device_health()
+    end
   end
 
   def get_latest_health(device_id) do

lib/nerves_hub/devices.ex

@@ -7,6 +7,7 @@ defmodule NervesHub.Devices.Device do
   alias NervesHub.Deployments.Deployment
   alias NervesHub.Devices.DeviceCertificate
   alias NervesHub.Devices.DeviceConnection
+  alias NervesHub.Devices.DeviceHealth
   alias NervesHub.Devices.DeviceMetric
   alias NervesHub.Extensions.DeviceExtensionsSetting
   alias NervesHub.Firmwares.FirmwareMetadata
@@ -37,6 +38,7 @@ defmodule NervesHub.Devices.Device do
     belongs_to(:product, Product)
     belongs_to(:deployment, Deployment)
     belongs_to(:latest_connection, DeviceConnection, type: :binary_id)
+    belongs_to(:latest_health, DeviceHealth)
 
     has_many(:device_certificates, DeviceCertificate, on_delete: :delete_all)
     has_many(:device_connections, DeviceConnection, on_delete: :delete_all)

lib/nerves_hub/devices/device.ex

@@ -7,16 +7,25 @@ defmodule NervesHub.Devices.DeviceHealth do
 
   @type t :: %__MODULE__{}
   @required_params [:device_id, :data]
+  @optional_params [:status, :status_reasons]
 
   schema "device_health" do
     belongs_to(:device, Device)
     field(:data, :map)
+
+    field(:status, Ecto.Enum,
+      values: [:unknown, :healthy, :warning, :unhealthy],
+      default: :unknown
+    )
+
+    field(:status_reasons, :map)
+
     timestamps(type: :utc_datetime_usec, updated_at: false)
   end
 
   def save(params) do
     %__MODULE__{}
-    |> cast(params, @required_params)
+    |> cast(params, @required_params ++ @optional_params)
     |> validate_required(@required_params)
   end
 end

lib/nerves_hub/devices/device_health.ex

@@ -38,6 +38,10 @@ defmodule NervesHub.Devices.Filtering do
     end
   end
 
+  def filter(query, _filters, :health_status, value) do
+    where(query, [latest_health: lh], lh.status == ^value)
+  end
+
   def filter(query, _filters, :connection, value) do
     if value == "not_seen" do
       where(query, [d], d.status == :registered)