diff --git a/.cspell.json b/.cspell.json index 7fcbcd91d..c6213df15 100644 --- a/.cspell.json +++ b/.cspell.json @@ -13,6 +13,13 @@ "test" ], "words": [ + "AEXAMPLE", + "BUCKETPOLICY", + "FABBEXAMPLE", + "Mjml", + "Opps", + "Spex", + "Ueberauth", "acreateserial", "anotherdummytoken", "behaviours", @@ -110,6 +117,7 @@ "progressbar", "psql", "rebar", + "reconnections", "registered", "rgba", "sendfile", diff --git a/assets/ui-rework/app.js b/assets/ui-rework/app.js index a194e4d93..8510b4740 100644 --- a/assets/ui-rework/app.js +++ b/assets/ui-rework/app.js @@ -8,6 +8,7 @@ import TimeAgo from "javascript-time-ago" import en from "javascript-time-ago/locale/en" import Chart from "./hooks/chart.js" +import ConnectedDevicesAnalytics from "./hooks/connectedDevicesAnalytics.js" import Console from "./hooks/console.js" import DeviceLocationMap from "./hooks/deviceLocationMap.js" import DeviceLocationMapWithGeocoder from "./hooks/deviceLocationMapWithGeocoder.js" @@ -33,6 +34,7 @@ let liveSocket = new LiveSocket("/live", Socket, { params: { _csrf_token: csrfToken }, hooks: { Chart, + ConnectedDevicesAnalytics, Console, DeviceLocationMap, DeviceLocationMapWithGeocoder, diff --git a/assets/ui-rework/hooks/connectedDevicesAnalytics.js b/assets/ui-rework/hooks/connectedDevicesAnalytics.js new file mode 100644 index 000000000..cda00d41a --- /dev/null +++ b/assets/ui-rework/hooks/connectedDevicesAnalytics.js @@ -0,0 +1,131 @@ +import Chart from "chart.js/auto" + +export default { + mounted() { + let unit = this.el.dataset.unit + + let label_singual = this.el.dataset.labelSingual + let label_plural = this.el.dataset.labelPlural + + let dataset = JSON.parse(this.el.dataset.metrics) + + const ctx = this.el + + var data = [] + for (let i = 0; i < dataset.length; i++) { + data.push({ + x: dataset[i].timestamp, + y: dataset[i].count + }) + } + + var gradient = ctx.getContext("2d").createLinearGradient(0, 0, 0, 400) + gradient.addColorStop(0, "rgba(99, 102, 241, 1)") + gradient.addColorStop(1, "rgba(99, 102, 241, 0)") + + const barChartDataset = { + type: "bar", + data: { + datasets: [ + { + backgroundColor: gradient, + hoverBackgroundColor: "#7f9cf5", + barPercentage: 0.75, + minBarLength: 2, + data: data + } + ] + }, + options: { + plugins: { + title: { + display: false + }, + legend: { + display: false, + labels: { + display: false + } + }, + tooltip: { + callbacks: { + title: function(context) { + date = new Date(context[0].parsed.x) + return date.toLocaleTimeString("en-NZ") + }, + label: function(context) { + if (context.raw.y === 1) { + unit = label_singual + } else { + unit = label_plural + } + return " " + context.formattedValue + " " + unit + } + } + } + }, + scales: { + x: { + display: false, + // grid: { + // display: false, + // drawOnChartArea: false, + // drawTicks: false + // }, + type: "time", + time: { + unit: unit, + displayFormats: { + millisecond: "HH:mm:ss.SSS", + second: "HH:mm:ss", + minute: "HH:mm", + hour: "HH:mm" + } + }, + ticks: { + display: false + } + }, + y: { + offset: false, + grid: { + display: false + }, + type: "linear", + min: 0, + // max: max + ticks: { + display: false + } + } + }, + responsive: true, + maintainAspectRatio: false, + layout: { + autoPadding: false, + padding: { + top: 0, + right: 10, + bottom: 0, + left: 5 + } + } + } + } + + const chart = new Chart(ctx, barChartDataset) + this.el.chart = chart + + this.handleEvent("update-charts", function(payload) { + if (payload.type == type) { + chart.data.datasets[0].data = payload.data + chart.update() + } + }) + + this.handleEvent("update-time-unit", function(payload) { + chart.options.scales.x.time.unit = payload.unit + chart.update() + }) + } +} diff --git a/config/runtime.exs b/config/runtime.exs index e52ced0d7..3f8f78876 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -39,6 +39,7 @@ config :nerves_hub, String.to_integer(System.get_env("DEPLOYMENT_CALCULATOR_INTERVAL_SECONDS", "3600")), mapbox_access_token: System.get_env("MAPBOX_ACCESS_TOKEN"), dashboard_enabled: System.get_env("DASHBOARD_ENABLED", "false") == "true", + insights_enabled: System.get_env("INSIGHTS_ENABLED", "false") == "true", extension_config: [ geo: [ # No interval, fetch geo on device connection by default diff --git a/lib/nerves_hub/devices/filtering.ex b/lib/nerves_hub/devices/filtering.ex index bec152173..581ad562e 100644 --- a/lib/nerves_hub/devices/filtering.ex +++ b/lib/nerves_hub/devices/filtering.ex @@ -6,6 +6,7 @@ defmodule NervesHub.Devices.Filtering do import Ecto.Query alias NervesHub.Devices.Alarms + alias NervesHub.Devices.Device alias NervesHub.Devices.DeviceMetric alias NervesHub.Types.Tag @@ -62,6 +63,25 @@ defmodule NervesHub.Devices.Filtering do ) end + def filter(query, _filters, :filter, "not_recently_seen") do + query + |> where([latest_connection: lc], lc.status == :disconnected) + |> where([latest_connection: lc], lc.disconnected_at < ago(24, "hour")) + end + + def filter(query, _filters, :filter, "unstable_connections") do + unstable_connections = + Device + |> select([d], %{id: d.id, count: count()}) + |> join(:left, [d], dc in assoc(d, :device_connections)) + |> where([_, dc], dc.established_at > ago(24, "hour")) + |> group_by([d], d.id) + + join(query, :inner, [d], uc in subquery(unstable_connections), + on: d.id == uc.id and uc.count > 5 + ) + end + def filter(query, _filters, :firmware_version, value) do where(query, [d], d.firmware_metadata["version"] == ^value) end diff --git a/lib/nerves_hub/firmwares.ex b/lib/nerves_hub/firmwares.ex index ffc535a66..c01ece0ff 100644 --- a/lib/nerves_hub/firmwares.ex +++ b/lib/nerves_hub/firmwares.ex @@ -68,6 +68,28 @@ defmodule NervesHub.Firmwares do |> Repo.all() end + @spec get_installed_firmwares(Product.t(), [String.t()]) :: [Firmware.t()] + def get_installed_firmwares(product, uuids) do + subquery = + Device + |> select([d], %{ + firmware_uuid: fragment("? ->> 'uuid'", d.firmware_metadata), + install_count: count(fragment("? ->> 'uuid'", d.firmware_metadata), :distinct) + }) + |> where([d], not is_nil(d.firmware_metadata)) + |> where([d], not is_nil(fragment("? ->> 'uuid'", d.firmware_metadata))) + |> Repo.exclude_deleted() + |> group_by([d], fragment("? ->> 'uuid'", d.firmware_metadata)) + + Firmware + |> join(:left, [f], d in subquery(subquery), on: d.firmware_uuid == f.uuid) + |> where([f], f.product_id == ^product.id) + |> where([f], f.uuid in ^uuids) + |> select_merge([_f, d], %{install_count: d.install_count}) + |> order_by([_f, d], d.install_count) + |> Repo.all() + end + @spec filter(Product.t(), map()) :: {[Product.t()], Flop.Meta.t()} def filter(product, opts \\ %{}) do opts = Map.reject(opts, fn {_key, val} -> is_nil(val) end) @@ -94,15 +116,25 @@ defmodule NervesHub.Firmwares do |> group_by([d], fragment("? ->> 'uuid'", d.firmware_metadata)) Firmware - |> join(:left, [f], d in subquery(subquery), on: d.firmware_uuid == f.uuid) + |> join(:left, [f], d in subquery(subquery), + as: :install_count, + on: d.firmware_uuid == f.uuid + ) |> where([f], f.product_id == ^product.id) + |> filter_selection(opts[:filter]) |> sort_firmware(sort_opts) - |> select_merge([_f, d], %{install_count: d.install_count}) + |> select_merge([_f, install_count: ic], %{install_count: ic.install_count}) |> Flop.run(flop) end + defp filter_selection(query, "active_firmware") do + where(query, [_f, install_count: ic], ic.install_count > 0) + end + + defp filter_selection(query, _filter), do: query + defp sort_firmware(query, {direction, :install_count}) do - order_by(query, [_f, d], {^direction, d.install_count}) + order_by(query, [_f, install_count: ic], {^direction, ic.install_count}) end defp sort_firmware(query, sort), do: order_by(query, ^sort) diff --git a/lib/nerves_hub/insights.ex b/lib/nerves_hub/insights.ex new file mode 100644 index 000000000..4e5a923c7 --- /dev/null +++ b/lib/nerves_hub/insights.ex @@ -0,0 +1,148 @@ +defmodule NervesHub.Insights do + @moduledoc """ + Queries for Product Insights. + + Some useful reads, which may inspire future improvements: + - https://danschultzer.com/posts/normalize-chart-data-using-ecto-postgresql + - https://danschultzer.com/posts/echarts-phoenix-liveview + """ + import Ecto.Query + + alias NervesHub.Devices.Device + alias NervesHub.Devices.DeviceConnection + alias NervesHub.Products.Product + + alias NervesHub.Repo + + @doc """ + Count of currently connected Devices, scoped to an individual Product. + """ + @spec connected_count(product :: Product.t()) :: number() + def connected_count(product) do + Device + |> join(:left, [d], lc in assoc(d, :latest_connection), as: :latest_connection) + |> where([d], d.product_id == ^product.id) + |> where([_, latest_connection: lc], lc.status == :connected) + |> Repo.aggregate(:count) + end + + @doc """ + Count of connected Devices over the last 24 hours, in 1 minute intervals, scoped to an individual Product. + """ + @spec connected_device_counts_last_24_hours(product :: Product.t()) :: list() + def connected_device_counts_last_24_hours(product) do + device_counts_query = + DeviceConnection + |> select([dc], %{ + device_count: count(dc.device_id, :distinct) + }) + |> where([dc], dc.product_id == ^product.id) + |> where( + [dc], + fragment( + "? BETWEEN DATE_BIN('1 minute'::interval, ?, 'epoch') AND DATE_BIN('1 minute'::interval, ?, 'epoch') + '1 minute'::interval", + dc.established_at, + parent_as(:series).timestamp, + parent_as(:series).timestamp + ) or + fragment( + "? BETWEEN DATE_BIN('1 minute'::interval, ?, 'epoch') AND DATE_BIN('1 minute'::interval, ?, 'epoch') + '1 minute'::interval", + dc.disconnected_at, + parent_as(:series).timestamp, + parent_as(:series).timestamp + ) or + (fragment( + "? < DATE_BIN('1 minute'::interval, ?, 'epoch')", + dc.established_at, + parent_as(:series).timestamp + ) and + fragment( + "? > DATE_BIN('1 minute'::interval, ?, 'epoch') + '1 minute'::interval", + dc.disconnected_at, + parent_as(:series).timestamp + )) or + (fragment( + "? < DATE_BIN('1 minute'::interval, ?, 'epoch')", + dc.established_at, + parent_as(:series).timestamp + ) and is_nil(dc.disconnected_at)) + ) + |> group_by([dc], dc.product_id) + + from(fragment("GENERATE_SERIES(NOW() - '1 hour'::interval, NOW(), '1 minute'::interval)"), + as: :series + ) + |> join(:left_lateral, [], subquery(device_counts_query), on: true, as: :counts) + |> select([gs, counts: dc], %{ + timestamp: + fragment("DATE_BIN('1 minute'::interval, ?, 'epoch')", gs) |> selected_as(:timestamp), + count: coalesce(dc.device_count, 0) + }) + |> order_by([series: gs], asc: gs.timestamp) + |> Repo.all() + end + + @doc """ + Count of devices where the health status isn't 'healthy' or 'unknown', scoped to an individual Product. + """ + @spec not_healthy_devices_count(product :: Product.t()) :: number() + def not_healthy_devices_count(product) do + Device + |> join(:left, [d], lc in assoc(d, :latest_health), as: :latest_health) + |> where([d], d.product_id == ^product.id) + |> where([_, latest_health: lh], lh.status in [:warning, :unhealthy]) + |> Repo.aggregate(:count) + end + + @doc """ + Count of devices which haven't connected in the last 24 hours, scoped to an individual Product. + """ + @spec not_recently_seen_count(product :: Product.t()) :: number() + def not_recently_seen_count(product) do + Device + |> join(:left, [d], lc in assoc(d, :latest_connection), as: :latest_connection) + |> where([d], d.product_id == ^product.id) + |> where([_, latest_connection: lc], lc.status == :disconnected) + |> where([_, latest_connection: lc], lc.disconnected_at < ago(24, "hour")) + |> Repo.aggregate(:count) + end + + @doc """ + Count of different firmware versions run on connected devices, scoped to an individual Product. + """ + @spec active_firmware_versions_count(product :: Product.t()) :: number() + def active_firmware_versions_count(product) do + Device + |> select([d], %{version: d.firmware_metadata["uuid"]}) + |> distinct(true) + |> where([d], d.product_id == ^product.id) + |> Repo.aggregate(:count) + end + + @doc """ + List of unique firmware versions run on connected devices, scoped to an individual Product. + """ + @spec active_firmware_versions(product :: Product.t()) :: list() + def active_firmware_versions(product) do + Device + |> select([d], %{version: d.firmware_metadata["uuid"]}) + |> distinct(true) + |> where([d], d.product_id == ^product.id) + |> Repo.all() + end + + @doc """ + Count of devices which have reconnected more than 5 times in the last 24 hours, scoped to an individual Product. + """ + @spec high_reconnect_count(product :: Product.t()) :: number() + def high_reconnect_count(product) do + Device + |> join(:left, [d], lc in assoc(d, :device_connections), as: :device_connections) + |> where([d], d.product_id == ^product.id) + |> where([_, device_connections: dc], dc.established_at > ago(24, "hour")) + |> group_by([d], d.id) + |> having([d, device_connections: dc], count(dc.id) > 5) + |> subquery() + |> Repo.aggregate(:count) + end +end diff --git a/lib/nerves_hub_web.ex b/lib/nerves_hub_web.ex index 28c1bb2a2..c82ef5200 100644 --- a/lib/nerves_hub_web.ex +++ b/lib/nerves_hub_web.ex @@ -96,6 +96,15 @@ defmodule NervesHubWeb do alias NervesHubWeb.Components.Navigation + @type tab :: + :archives + | :deployments + | :devices + | :firmware + | :insights + | :settings + | :support_scripts + # Routes generation with the ~p sigil unquote(verified_routes()) @@ -109,10 +118,7 @@ defmodule NervesHubWeb do def page_title(socket, page_title), do: assign(socket, :page_title, page_title) - @spec sidebar_tab( - Phoenix.Socket.t(), - :archives | :firmware | :deployments | :devices | :settings | :support_scripts - ) :: Phoenix.Socket.t() + @spec sidebar_tab(Phoenix.LiveView.Socket.t(), tab()) :: Phoenix.LiveView.Socket.t() def sidebar_tab(socket, tab) do socket |> assign(:sidebar_tab, tab) diff --git a/lib/nerves_hub_web/components/navigation.ex b/lib/nerves_hub_web/components/navigation.ex index 30c973116..a3205260d 100644 --- a/lib/nerves_hub_web/components/navigation.ex +++ b/lib/nerves_hub_web/components/navigation.ex @@ -15,6 +15,14 @@ defmodule NervesHubWeb.Components.Navigation do def updated_sidebar(assigns) do ~H"""