Add system average load and memory metrics (#50)

* Add system load average and memory metrics to the exporter.

* Add System Load Average and System Memory panels to the Grafana dashboard and update panel layout.

Author: marcinbudny

README.md

@@ -260,6 +260,20 @@ eventstore_tcp_connection_received_bytes{client_name="ES-8d3d28dd-7944-49b4-a130
 # TYPE eventstore_tcp_connection_sent_bytes counter
 eventstore_tcp_connection_sent_bytes{client_name="ES-8d3d28dd-7944-49b4-a130-686cc9aecc92",external="true",id="7d7f9762-9565-4cca-8bdf-2dc7c845c19a",local_endpoint="172.16.1.11:1113",remote_endpoint="172.16.1.1:59230",ssl="false"} 252635
 
+# HELP eventstore_sys_free_memory_bytes System free memory in bytes
+# TYPE eventstore_sys_free_memory_bytes gauge
+eventstore_sys_free_memory_bytes 2.502856704e+09
+
+# HELP eventstore_sys_total_memory_bytes System total memory in bytes
+# TYPE eventstore_sys_total_memory_bytes gauge
+eventstore_sys_total_memory_bytes 1.6656265216e+10
+
+# HELP eventstore_sys_loadavg System load average
+# TYPE eventstore_sys_loadavg gauge
+eventstore_sys_loadavg{period="15m"} 0.35
+eventstore_sys_loadavg{period="1m"} 0.72
+eventstore_sys_loadavg{period="5m"} 0.52
+
 # HELP eventstore_up Whether the EventStore scrape was successful
 # TYPE eventstore_up gauge
 eventstore_up 1

internal/client/server_stats.go

@@ -31,7 +31,16 @@ type TCPStats struct {
 }
 
 type SystemStats struct {
-	Drives map[string]DriveStats `json:"drive"`
+	Drives   map[string]DriveStats `json:"drive"`
+	LoadAvg  LoadAvgStats          `json:"loadavg"`
+	FreeMem  int64                 `json:"freeMem"`
+	TotalMem int64                 `json:"totalMem"`
+}
+
+type LoadAvgStats struct {
+	OneMin     float64 `json:"1m"`
+	FiveMin    float64 `json:"5m"`
+	FifteenMin float64 `json:"15m"`
 }
 
 type DriveStats struct {

internal/collector/collector.go

@@ -39,6 +39,10 @@ type Collector struct {
 	driveTotalBytes     *prometheus.Desc
 	driveAvailableBytes *prometheus.Desc
 
+	sysLoadAvg          *prometheus.Desc
+	sysFreeMemoryBytes  *prometheus.Desc
+	sysTotalMemoryBytes *prometheus.Desc
+
 	projectionRunning                     *prometheus.Desc
 	projectionStatus                      *prometheus.Desc
 	projectionProgress                    *prometheus.Desc
@@ -92,6 +96,10 @@ func NewCollector(config *config.Config, client *client.EventStoreStatsClient) *
 		driveTotalBytes:     prometheus.NewDesc("eventstore_drive_total_bytes", "Drive total size in bytes", []string{"drive"}, nil),
 		driveAvailableBytes: prometheus.NewDesc("eventstore_drive_available_bytes", "Drive available bytes", []string{"drive"}, nil),
 
+		sysLoadAvg:          prometheus.NewDesc("eventstore_sys_loadavg", "System load average", []string{"period"}, nil),
+		sysFreeMemoryBytes:  prometheus.NewDesc("eventstore_sys_free_memory_bytes", "System free memory in bytes", nil, nil),
+		sysTotalMemoryBytes: prometheus.NewDesc("eventstore_sys_total_memory_bytes", "System total memory in bytes", nil, nil),
+
 		projectionRunning:                     prometheus.NewDesc("eventstore_projection_running", "If 1, projection is in 'Running' state", []string{"projection"}, nil),
 		projectionStatus:                      prometheus.NewDesc("eventstore_projection_status", "If 1, projection is in specified state", []string{"projection", "status"}, nil),
 		projectionProgress:                    prometheus.NewDesc("eventstore_projection_progress", "Projection progress 0 - 1, where 1 = projection progress at 100%", []string{"projection"}, nil),
@@ -144,6 +152,10 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
 	ch <- c.driveTotalBytes
 	ch <- c.driveAvailableBytes
 
+	ch <- c.sysLoadAvg
+	ch <- c.sysFreeMemoryBytes
+	ch <- c.sysTotalMemoryBytes
+
 	ch <- c.projectionRunning
 	ch <- c.projectionStatus
 	ch <- c.projectionProgress
@@ -190,6 +202,7 @@ func (c *Collector) collectFromStats(ch chan<- prometheus.Metric, stats *client.
 	c.collectFromTCPConnectionStats(ch, stats.TCPConnections)
 	c.collectFromQueueStats(ch, stats.Server.Es.Queues)
 	c.collectFromDriveStats(ch, stats.Server.System.Drives)
+	c.collectFromSystemStats(ch, stats.Server.System)
 	c.collectFromProjectionStats(ch, stats.Projections)
 	c.collectFromSubscriptionStats(ch, stats.Subscriptions)
 	c.collectFromStreamStats(ch, stats.Streams)
@@ -240,6 +253,15 @@ func (c *Collector) collectFromDriveStats(ch chan<- prometheus.Metric, stats map
 	}
 }
 
+func (c *Collector) collectFromSystemStats(ch chan<- prometheus.Metric, stats client.SystemStats) {
+	ch <- prometheus.MustNewConstMetric(c.sysLoadAvg, prometheus.GaugeValue, stats.LoadAvg.OneMin, "1m")
+	ch <- prometheus.MustNewConstMetric(c.sysLoadAvg, prometheus.GaugeValue, stats.LoadAvg.FiveMin, "5m")
+	ch <- prometheus.MustNewConstMetric(c.sysLoadAvg, prometheus.GaugeValue, stats.LoadAvg.FifteenMin, "15m")
+
+	ch <- prometheus.MustNewConstMetric(c.sysFreeMemoryBytes, prometheus.GaugeValue, float64(stats.FreeMem))
+	ch <- prometheus.MustNewConstMetric(c.sysTotalMemoryBytes, prometheus.GaugeValue, float64(stats.TotalMem))
+}
+
 func (c *Collector) collectFromProjectionStats(ch chan<- prometheus.Metric, stats []client.ProjectionStats) {
 	for _, projection := range stats {
 		running := 0.0

internal/server/server_basic_test.go

@@ -30,6 +30,9 @@ func Test_BasicMetrics(t *testing.T) {
 	assertHasMetric(t, metrics, "eventstore_disk_io_write_ops", "gauge")
 	assertHasMetric(t, metrics, "eventstore_disk_io_written_bytes", "gauge")
 	assertHasMetric(t, metrics, "eventstore_drive_available_bytes", "gauge")
+	assertHasMetric(t, metrics, "eventstore_sys_loadavg", "gauge")
+	assertHasMetric(t, metrics, "eventstore_sys_free_memory_bytes", "gauge")
+	assertHasMetric(t, metrics, "eventstore_sys_total_memory_bytes", "gauge")
 	assertHasMetric(t, metrics, "eventstore_process_cpu", "gauge")
 	assertHasMetric(t, metrics, "eventstore_process_memory_bytes", "gauge")
 	assertHasMetric(t, metrics, "eventstore_queue_items_processed_total", "counter")