Merge pull request vpenso#13 from rug-cit-hpc/master

Add metrics for CPUs (extracted from Slurm sinfo)

Author: mtds

Makefile

@@ -2,7 +2,7 @@ PROJECT_NAME = prometheus-slurm-exporter
 ifndef GOPATH
 	GOPATH=$(shell pwd):/usr/share/gocode
 endif
-GOFILES=main.go nodes.go queue.go scheduler.go
+GOFILES=cpus.go main.go nodes.go queue.go scheduler.go
 GOBIN=bin/$(PROJECT_NAME)
 
 build:

README.md

@@ -4,6 +4,15 @@ Prometheus collector and exporter for metrics extracted from the [Slurm](https:/
 
 ## Exported Metrics
 
+### State of the CPUs
+
+* **Allocated**: CPUs which have been allocated to a job.
+* **Idle**: CPUs not allocated to a job and thus available for use.
+* **Other**: CPUs which are unavailable for use at the moment.
+* **Total**: total number of CPUs.
+
+[Information extracted from the SLURM **sinfo** command](https://slurm.schedmd.com/sinfo.html)
+
 ### State of the Nodes
 
 * **Allocated**: nodes which has been allocated to one or more jobs.

cpus.go

@@ -0,0 +1,102 @@
+/* Copyright 2017 Victor Penso, Matteo Dessalvi
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>. */
+
+package main
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"io/ioutil"
+	"log"
+	"os/exec"
+	"strconv"
+	"strings"
+)
+
+type CPUsMetrics struct {
+	alloc float64
+	idle  float64
+	other float64
+	total float64
+}
+
+func CPUsGetMetrics() *CPUsMetrics {
+	return ParseCPUsMetrics(CPUsData())
+}
+
+func ParseCPUsMetrics(input []byte) *CPUsMetrics {
+	var cm CPUsMetrics
+	if strings.Contains(string(input), "/") {
+	    splitted := strings.Split(strings.TrimSpace(string(input)), "/")
+	    cm.alloc, _ = strconv.ParseFloat(splitted[0], 64)
+	    cm.idle, _  = strconv.ParseFloat(splitted[1], 64)
+	    cm.other, _ = strconv.ParseFloat(splitted[2], 64)
+	    cm.total, _ = strconv.ParseFloat(splitted[3], 64)
+	} 
+	return &cm
+}
+
+// Execute the sinfo command and return its output
+func CPUsData() []byte {
+	cmd := exec.Command("sinfo", "-h", "-o %C")
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		log.Fatal(err)
+	}
+	if err := cmd.Start(); err != nil {
+		log.Fatal(err)
+	}
+	out, _ := ioutil.ReadAll(stdout)
+	if err := cmd.Wait(); err != nil {
+		log.Fatal(err)
+	}
+	return out
+}
+
+/*
+ * Implement the Prometheus Collector interface and feed the
+ * Slurm scheduler metrics into it.
+ * https://godoc.org/github.com/prometheus/client_golang/prometheus#Collector
+ */
+
+func NewCPUsCollector() *CPUsCollector {
+	return &CPUsCollector{
+		alloc: prometheus.NewDesc("slurm_cpus_alloc", "Allocated CPUs", nil, nil),
+		idle:  prometheus.NewDesc("slurm_cpus_idle", "Idle CPUs", nil, nil),
+		other: prometheus.NewDesc("slurm_cpus_other", "Mix CPUs", nil, nil),
+		total: prometheus.NewDesc("slurm_cpus_total", "Total CPUs", nil, nil),
+	}
+}
+
+type CPUsCollector struct {
+	alloc *prometheus.Desc
+	idle  *prometheus.Desc
+	other *prometheus.Desc
+	total *prometheus.Desc
+}
+
+// Send all metric descriptions
+func (cc *CPUsCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- cc.alloc
+	ch <- cc.idle
+	ch <- cc.other
+	ch <- cc.total
+}
+func (cc *CPUsCollector) Collect(ch chan<- prometheus.Metric) {
+	cm := CPUsGetMetrics()
+	ch <- prometheus.MustNewConstMetric(cc.alloc, prometheus.GaugeValue, cm.alloc)
+	ch <- prometheus.MustNewConstMetric(cc.idle,  prometheus.GaugeValue, cm.idle)
+	ch <- prometheus.MustNewConstMetric(cc.other, prometheus.GaugeValue, cm.other)
+	ch <- prometheus.MustNewConstMetric(cc.total, prometheus.GaugeValue, cm.total)
+}

cpus_test.go

@@ -0,0 +1,34 @@
+/* Copyright 2017 Victor Penso, Matteo Dessalvi
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>. */
+
+package main
+
+import (
+  "testing"
+  "os"
+  "io/ioutil"
+)
+
+func TestCPUsMetrics(t *testing.T) {
+  // Read the input data from a file
+  file, err := os.Open("test_data/sinfo_cpus.txt")
+  if err != nil { t.Fatalf("Can not open test data: %v", err) }
+  data, err := ioutil.ReadAll(file)
+  t.Logf("%+v", ParseCPUsMetrics(data))
+}
+
+func TestCPUssGetMetrics(t *testing.T) {
+  t.Logf("%+v", CPUsGetMetrics())
+}

main.go

@@ -28,6 +28,7 @@ func init() {
   prometheus.MustRegister(NewSchedulerCollector()) // from scheduler.go
   prometheus.MustRegister(NewQueueCollector())     // from queue.go
   prometheus.MustRegister(NewNodesCollector())     // from nodes.go
+  prometheus.MustRegister(NewCPUsCollector())      // from cpus.go
 }
 
 var listenAddress = flag.String(

test_data/sinfo_cpus.txt

@@ -0,0 +1 @@
+5725/877/34/6636