Add device ICMP statistics for alerting (librenms#17041)

* Add packet loss column to the devices table

* Local fix to allow me to roll bac schema

* Create a new device_stats table to store dynamic stats for devices.  Initial data is some RTT and packet loss stats/

* Formatting fix

* Update DB schema

* Remove unneeded migration

* Remove refernce to device loss column

* Added some documentation around the averaging factor option

* Update to use LibrenmsConfig

* Add drop if exists on schema migration

* Remove duplicate code

* Include the device ID when creating a new devicestats object

* Allow nulls in stats columns

* Refactor DeviceStats to extend DeviceRelatedModel

Updated DeviceStats to extend DeviceRelatedModel and removed the device relationship method.

* Remove unused BelongsTo import in DeviceStats

* Change ping RTT and loss fields to nullable

* Added the device_stats to the alert entities documentat

---------

Co-authored-by: Tony Murray <[email protected]>

Author: eskyuu

LibreNMS/Data/Source/FpingResponse.php

@@ -29,6 +29,7 @@
 use App\Facades\LibrenmsConfig;
 use App\Facades\Rrd;
 use App\Models\Device;
+use App\Models\DeviceStats;
 use Carbon\Carbon;
 use LibreNMS\Exceptions\FpingUnparsableLine;
 use LibreNMS\RRD\RrdDefinition;
@@ -141,6 +142,25 @@ public function __toString(): string
 
     public function saveStats(Device $device): void
     {
+        $stats = $device->stats ?? new DeviceStats(['device_id' => $device->device_id]);
+        $stats->ping_last_timestamp = Carbon::now();
+        // Only update the latency if we have data
+        if ($this->avg_latency) {
+            $stats->ping_rtt_prev = $stats->ping_rtt_last ?: $this->avg_latency;
+            $stats->ping_rtt_last = $this->avg_latency;
+            // Average is calcualted as the exponential weighted moving average
+            $stats->ping_rtt_avg = $stats->ping_rtt_avg ? $stats->ping_rtt_avg + (($stats->ping_rtt_last - $stats->ping_rtt_avg) * LibrenmsConfig::get('device_stats_avg_factor')) : $stats->ping_rtt_last;
+        }
+        // Only update loss if we transmitted a packet
+        if ($this->transmitted) {
+            $stats->ping_loss_prev = $stats->ping_loss_last ?: 100 * ($this->transmitted - $this->received) / $this->transmitted;
+            $stats->ping_loss_last = 100 * ($this->transmitted - $this->received) / $this->transmitted;
+            // Average is calcualted as the exponential weighted moving average
+            $stats->ping_loss_avg = $stats->ping_loss_avg ? $stats->ping_loss_avg + (($stats->ping_loss_last - $stats->ping_loss_avg) * LibrenmsConfig::get('device_stats_avg_factor')) : $stats->ping_loss_last;
+        }
+        $stats->save();
+
+        // Update the stats stored in the device table until the field is removed
         $device->last_ping = Carbon::now();
         $device->last_ping_timetaken = $this->avg_latency ?: $device->last_ping_timetaken;
         $device->save();

app/Models/Device.php

@@ -1180,6 +1180,14 @@ public function services(): HasMany
         return $this->hasMany(Service::class, 'device_id');
     }
 
+    /**
+     * @return \Illuminate\Database\Eloquent\Relations\HasOne<\App\Models\DeviceStats, $this>
+     */
+    public function stats(): HasOne
+    {
+        return $this->hasOne(DeviceStats::class, 'device_id');
+    }
+
     /**
      * @return \Illuminate\Database\Eloquent\Relations\HasMany<\App\Models\Storage, $this>
      */

app/Models/DeviceStats.php

@@ -0,0 +1,21 @@
+<?php
+
+namespace App\Models;
+
+use Illuminate\Database\Eloquent\Factories\HasFactory;
+
+class DeviceStats extends DeviceRelatedModel
+{
+    use HasFactory;
+
+    protected $fillable = [
+        'device_id',
+        'ping_last_timestamp',
+        'ping_rtt_last',
+        'ping_rtt_prev',
+        'ping_rtt_avg',
+        'ping_loss_last',
+        'ping_loss_prev',
+        'ping_loss_avg',
+    ];
+}

database/migrations/2025_05_28_092148_create_device_stats.php

@@ -0,0 +1,39 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        Schema::dropIfExists('device_stats');
+        Schema::create('device_stats', function (Blueprint $table) {
+            $table->id();
+            $table->timestamps();
+            $table->unsignedInteger('device_id')->unique();
+            $table->timestamp('ping_last_timestamp');
+            $table->float('ping_rtt_last')->unsigned()->nullable();
+            $table->float('ping_rtt_prev')->unsigned()->nullable();
+            $table->float('ping_rtt_avg')->unsigned()->nullable();
+            $table->float('ping_rtt_diff_avg_last')->virtualAs('ping_rtt_last - ping_rtt_avg');
+            $table->float('ping_rtt_diff_prev_last')->virtualAs('ping_rtt_last - ping_rtt_prev');
+            $table->float('ping_loss_last')->unsigned()->nullable();
+            $table->float('ping_loss_prev')->unsigned()->nullable();
+            $table->float('ping_loss_avg')->unsigned()->nullable();
+            $table->foreign('device_id')->references('device_id')->on('devices')->onDelete('cascade');
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        Schema::dropIfExists('device_stats');
+    }
+};

doc/Alerting/Entities.md

@@ -25,6 +25,21 @@ Entity | Description
 `devices.last_polled` | The the last polled datetime (yyyy-mm-dd hh:mm:ss)
 `devices.type` | The device type such as network, server, firewall, etc.
 
+## Device Stats
+
+Entity | Description
+---|---
+`device_stats.ping_loss_last` | Packet loss at last poll (percent)
+`device_stats.ping_loss_avg` | Average packet loss (percent)
+`device_stats.ping_rtt_last` | Ping RTT at last poll (ms)
+`device_stats.ping_rtt_avg` | Average ping RTT (ms)
+`device_stats.ping_rtt_diff_avg_last` | Difference between the RTT last and average ping (ms)
+`device_stats.ping_rtt_diff_avg_last` | Difference between the RTT of the ping from the last 2 polls (ms)
+
+Details on how the averages above are calculated can be found [here](../Support/Configuration.md#averaging-factor).
+
+The difference fields can be used to detect when ping times increase from their normal valuse.
+
 ## BGP Peers
 
 Entity | Description

doc/Support/Configuration.md

@@ -1156,6 +1156,28 @@ is discovered.
     lnms config:set storage_perc_warn 60
     ```
 
+## Averaging Factor
+
+LibreNMS keeps track of average values in the database for some metrics so we
+can alert on changes (e.g. if the ping time increases from the average). To 
+achieve this goal we want the average to move slowly when there is a change
+to the values being recorded so there is time to alert, but we also need to
+eventually stop alerting if the average value becomes the new normal.
+
+The following configuration variable can be adjusted if you make use of the
+average values, and find that they either change too quickly or slowly. If
+you make this setting bigger (closer to 1) the averages will change faster,
+and if you make it smaller (closer to 0) the average will change slower.
+
+```bash
+lnms config:set device_stats_avg_factor 0.05
+```
+
+If you want to understand more about this, the device statistics uses an
+exponential weighted moving average function to update the average without
+needing to keep multiple values. You can look up independently if you want
+to understand more about this option.
+
 ## IRC Bot
 
 Please refer to [IRC Bot](../Extensions/IRC-Bot.md)

lang/en/settings.php

@@ -2364,6 +2364,10 @@
             'description' => 'Show devices dependecies on location map',
             'help' => 'Show links between devices on the location map based on parent dependencies',
         ],
+        'device_stats_avg_factor' => [
+            'description' => 'Averaging factor',
+            'help' => 'We calculate a moving average using an exponential weighted moving average function.  This is the factor used by the function to control how much the current value affects the average.  Values closer to 1 will make the average change quicker.',
+        ],
         'whois' => [
             'description' => 'Path to whois',
         ],

resources/definitions/config_definitions.json

@@ -1447,6 +1447,16 @@
                 "{{ $ip }}":  "ip"
             }
         },
+        "device_stats_avg_factor": {
+            "default": 0.05,
+            "group": "poller",
+            "section": "ping",
+            "order": 4,
+            "type": "float",
+            "validate": {
+                "value": "numeric|between:0.000001,0.9"
+            }
+        },
         "device_traffic_descr": {
             "default": [
                 "/loopback/",

resources/definitions/schema/db_schema.yaml

@@ -733,6 +733,26 @@ device_relationships:
   Constraints:
     device_relationship_child_device_id_fk: { name: device_relationship_child_device_id_fk, foreign_key: child_device_id, table: devices, key: device_id, extra: 'ON DELETE CASCADE' }
     device_relationship_parent_device_id_fk: { name: device_relationship_parent_device_id_fk, foreign_key: parent_device_id, table: devices, key: device_id, extra: 'ON DELETE CASCADE' }
+device_stats:
+  Columns:
+    - { Field: id, Type: 'bigint unsigned', 'Null': false, Extra: auto_increment }
+    - { Field: created_at, Type: timestamp, 'Null': true, Extra: '' }
+    - { Field: updated_at, Type: timestamp, 'Null': true, Extra: '' }
+    - { Field: device_id, Type: 'int unsigned', 'Null': false, Extra: '' }
+    - { Field: ping_last_timestamp, Type: timestamp, 'Null': false, Extra: '' }
+    - { Field: ping_rtt_last, Type: 'double unsigned', 'Null': true, Extra: '' }
+    - { Field: ping_rtt_prev, Type: 'double unsigned', 'Null': true, Extra: '' }
+    - { Field: ping_rtt_avg, Type: 'double unsigned', 'Null': true, Extra: '' }
+    - { Field: ping_rtt_diff_avg_last, Type: double, 'Null': true, Extra: 'VIRTUAL GENERATED' }
+    - { Field: ping_rtt_diff_prev_last, Type: double, 'Null': true, Extra: 'VIRTUAL GENERATED' }
+    - { Field: ping_loss_last, Type: 'double unsigned', 'Null': true, Extra: '' }
+    - { Field: ping_loss_prev, Type: 'double unsigned', 'Null': true, Extra: '' }
+    - { Field: ping_loss_avg, Type: 'double unsigned', 'Null': true, Extra: '' }
+  Indexes:
+    PRIMARY: { Name: PRIMARY, Columns: [id], Unique: true, Type: BTREE }
+    device_stats_device_id_unique: { Name: device_stats_device_id_unique, Columns: [device_id], Unique: true, Type: BTREE }
+  Constraints:
+    device_stats_device_id_foreign: { name: device_stats_device_id_foreign, foreign_key: device_id, table: devices, key: device_id, extra: 'ON DELETE CASCADE' }
 entityState:
   Columns:
     - { Field: entity_state_id, Type: 'int unsigned', 'Null': false, Extra: auto_increment }