Merge pull request #7 from stackhpc/bugfix/logs

Limit number of logs collected

Author: dougszumski

README.md

@@ -31,7 +31,21 @@ problematic the collection of logs can be configured as follows:
 To disable log collection you can set:
 
 ```yaml
-collectlogs: false
+hosts:
+  default:
+    username: username
+    password: password
+    collectlogs: false
+```
+
+or for a group:
+
+```yaml
+groups:
+  group1:
+    username: username
+    password: password
+    collectlogs: false
 ```
 
 2) via the `collectlogs` query parameter
@@ -60,6 +74,47 @@ curl '127.0.0.1:9123/redfish?target=10.10.12.23&collectlogs=true'
 
 The `collectlogs` query parameter can be included in Prometheus config.
 
+
+### Log collection count
+
+To restrict the number of logs collected for all log services:
+
+```
+hosts:
+  default:
+    username: username
+    password: password
+    collectlogs: true
+    logcount:
+      DEFAULT: 10
+```
+
+For a particular log service, `Sel`:
+
+```
+hosts:
+  default:
+    username: username
+    password: password
+    collectlogs: true
+    logcount:
+      Sel: 20
+```
+
+Collect all logs for one service, `Sel` , but limit others:
+
+```
+hosts:
+  default:
+    username: username
+    password: password
+    collectlogs: true
+    logcount:
+      DEFAULT: 10
+      Sel: -1
+```
+
+
 ## Building
 
 To build the redfish_exporter executable run the command:

collector/chassis_collector.go

@@ -7,8 +7,8 @@ import (
 	"sync"
 
 	"github.com/apex/log"
+	"github.com/jenningsloy318/redfish_exporter/common"
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/stmcginnis/gofish"
 	"github.com/stmcginnis/gofish/redfish"
 )
 
@@ -33,9 +33,8 @@ var (
 
 // ChassisCollector implements the prometheus.Collector.
 type ChassisCollector struct {
-	redfishClient         *gofish.APIClient
+	Ctx                   *common.CollectionContext
 	metrics               map[string]Metric
-	collectLogs           bool
 	collectorScrapeStatus *prometheus.GaugeVec
 	Log                   *log.Entry
 }
@@ -91,13 +90,12 @@ func createChassisMetricMap() map[string]Metric {
 }
 
 // NewChassisCollector returns a collector that collecting chassis statistics
-func NewChassisCollector(redfishClient *gofish.APIClient, collectLogs bool, logger *log.Entry) *ChassisCollector {
+func NewChassisCollector(ctx *common.CollectionContext, logger *log.Entry) *ChassisCollector {
 	// get service from redfish client
 
 	return &ChassisCollector{
-		redfishClient: redfishClient,
-		metrics:       chassisMetrics,
-		collectLogs:   collectLogs,
+		metrics: chassisMetrics,
+		Ctx:     ctx,
 		Log: logger.WithFields(log.Fields{
 			"collector": "ChassisCollector",
 		}),
@@ -124,8 +122,8 @@ func (c *ChassisCollector) Describe(ch chan<- *prometheus.Desc) {
 // Collect implemented prometheus.Collector
 func (c *ChassisCollector) Collect(ch chan<- prometheus.Metric) {
 	collectorLogContext := c.Log
-	collectLogs := c.collectLogs
-	service := c.redfishClient.Service
+	collectLogs := c.Ctx.CollectLogs
+	service := c.Ctx.RedfishClient.Service
 
 	// get a list of chassis from service
 	if chassises, err := service.Chassis(); err != nil {
@@ -252,7 +250,7 @@ func (c *ChassisCollector) Collect(ch chan<- prometheus.Metric) {
 					wg6.Add(len(logServices))
 
 					for _, logService := range logServices {
-						if err = parseLogService(ch, chassisMetrics, ChassisSubsystem, chassisID, logService, wg6); err != nil {
+						if err = parseLogService(ch, chassisMetrics, c.Ctx, chassisLogContext, ChassisSubsystem, chassisID, logService, wg6); err != nil {
 							chassisLogContext.WithField("operation", "chassis.LogServices()").WithError(err).Error("error getting log entries from log service")
 						}
 					}

collector/common_collector.go

@@ -4,7 +4,10 @@ import (
 	"fmt"
 	"sync"
 
+	"github.com/apex/log"
+	redfish_common "github.com/jenningsloy318/redfish_exporter/common"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/stmcginnis/gofish/common"
 	"github.com/stmcginnis/gofish/redfish"
 )
 
@@ -33,7 +36,7 @@ func addToMetricMap(metricMap map[string]Metric, subsystem, name, help string, v
 	}
 }
 
-func parseLogService(ch chan<- prometheus.Metric, metrics map[string]Metric, subsystem, collectorID string, logService *redfish.LogService, wg *sync.WaitGroup) (err error) {
+func parseLogService(ch chan<- prometheus.Metric, metrics map[string]Metric, ctx *redfish_common.CollectionContext, logger *log.Entry, subsystem, collectorID string, logService *redfish.LogService, wg *sync.WaitGroup) (err error) {
 	defer wg.Done()
 	logServiceName := logService.Name
 	logServiceID := logService.ID
@@ -50,8 +53,24 @@ func parseLogService(ch chan<- prometheus.Metric, metrics map[string]Metric, sub
 	if logServiceHealthStateValue, ok := parseCommonStatusHealth(logServiceHealthState); ok {
 		ch <- prometheus.MustNewConstMetric(metrics[fmt.Sprintf("%s_%s", subsystem, "log_service_health_state")].desc, prometheus.GaugeValue, logServiceHealthStateValue, logServiceLabelValues...)
 	}
+	var (
+		logEntries []*redfish.LogEntry
+	)
+	logCount, ok := ctx.LogCount[logServiceID]
+	if !ok {
+		logCount, ok = ctx.LogCount["DEFAULT"]
+		if !ok {
+			logCount = -1
+		}
+	}
+
+	logger.WithField("operation", "parseLogService").Info(fmt.Sprintf("logServiceID=%s, logcount=%d", logServiceID, logCount))
 
-	logEntries, err := logService.Entries()
+	if logCount > 0 {
+		logEntries, err = logService.FilteredEntries(common.WithTop(logCount))
+	} else {
+		logEntries, err = logService.Entries()
+	}
 	if err != nil {
 		return
 	}
@@ -65,7 +84,6 @@ func parseLogService(ch chan<- prometheus.Metric, metrics map[string]Metric, sub
 			continue
 		}
 		go parseLogEntry(ch, metrics[fmt.Sprintf("%s_%s", subsystem, "log_entry_severity_state")].desc, collectorID, logServiceName, logServiceID, logEntry, wg2)
-
 		processed[logEntry.MessageID] = true
 	}
 	return

collector/manager_collector.go

@@ -5,8 +5,8 @@ import (
 	"sync"
 
 	"github.com/apex/log"
+	"github.com/jenningsloy318/redfish_exporter/common"
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/stmcginnis/gofish"
 )
 
 // ManagerSubmanager is the manager subsystem
@@ -22,9 +22,8 @@ var (
 
 // ManagerCollector implements the prometheus.Collector.
 type ManagerCollector struct {
-	redfishClient         *gofish.APIClient
+	Ctx                   *common.CollectionContext
 	metrics               map[string]Metric
-	collectLogs           bool
 	collectorScrapeStatus *prometheus.GaugeVec
 	Log                   *log.Entry
 }
@@ -43,11 +42,10 @@ func createManagerMetricMap() map[string]Metric {
 }
 
 // NewManagerCollector returns a collector that collecting memory statistics
-func NewManagerCollector(redfishClient *gofish.APIClient, collectLogs bool, logger *log.Entry) *ManagerCollector {
+func NewManagerCollector(ctx *common.CollectionContext, logger *log.Entry) *ManagerCollector {
 	return &ManagerCollector{
-		redfishClient: redfishClient,
-		metrics:       managerMetrics,
-		collectLogs:   collectLogs,
+		metrics: managerMetrics,
+		Ctx:     ctx,
 		Log: logger.WithFields(log.Fields{
 			"collector": "ManagerCollector",
 		}),
@@ -74,9 +72,9 @@ func (m *ManagerCollector) Describe(ch chan<- *prometheus.Desc) {
 // Collect implemented prometheus.Collector
 func (m *ManagerCollector) Collect(ch chan<- prometheus.Metric) {
 	collectorLogContext := m.Log
-	collectLogs := m.collectLogs
+	collectLogs := m.Ctx.CollectLogs
 	//get service
-	service := m.redfishClient.Service
+	service := m.Ctx.RedfishClient.Service
 
 	// get a list of managers from service
 	if managers, err := service.Managers(); err != nil {
@@ -118,7 +116,7 @@ func (m *ManagerCollector) Collect(ch chan<- prometheus.Metric) {
 					wg.Add(len(logServices))
 
 					for _, logService := range logServices {
-						if err = parseLogService(ch, managerMetrics, ManagerSubmanager, ManagerID, logService, wg); err != nil {
+						if err = parseLogService(ch, managerMetrics, m.Ctx, managerLogContext, ManagerSubmanager, ManagerID, logService, wg); err != nil {
 							managerLogContext.WithField("operation", "manager.LogServices()").WithError(err).Error("error getting log entries from log service")
 						}
 					}

collector/redfish_collector.go

@@ -2,11 +2,11 @@ package collector
 
 import (
 	"bytes"
-	"fmt"
 	"sync"
 	"time"
 
-	"github.com/apex/log"
+	alog "github.com/apex/log"
+	"github.com/jenningsloy318/redfish_exporter/common"
 	"github.com/prometheus/client_golang/prometheus"
 	gofish "github.com/stmcginnis/gofish"
 	gofishcommon "github.com/stmcginnis/gofish/common"
@@ -40,22 +40,16 @@ type RedfishCollector struct {
 }
 
 // NewRedfishCollector return RedfishCollector
-func NewRedfishCollector(host string, username string, password string, collectLogs bool, logger *log.Entry) *RedfishCollector {
+func NewRedfishCollector(ctx *common.CollectionContext, logger *alog.Entry) *RedfishCollector {
 	var collectors map[string]prometheus.Collector
-	collectorLogCtx := logger
-	redfishClient, err := newRedfishClient(host, username, password)
-	if err != nil {
-		collectorLogCtx.WithError(err).Error("error creating redfish client")
-	} else {
-		chassisCollector := NewChassisCollector(redfishClient, collectLogs, collectorLogCtx)
-		systemCollector := NewSystemCollector(redfishClient, collectLogs, collectorLogCtx)
-		managerCollector := NewManagerCollector(redfishClient, collectLogs, collectorLogCtx)
+	chassisCollector := NewChassisCollector(ctx, logger)
+	systemCollector := NewSystemCollector(ctx, logger)
+	managerCollector := NewManagerCollector(ctx, logger)
 
-		collectors = map[string]prometheus.Collector{"chassis": chassisCollector, "system": systemCollector, "manager": managerCollector}
-	}
+	collectors = map[string]prometheus.Collector{"chassis": chassisCollector, "system": systemCollector, "manager": managerCollector}
 
 	return &RedfishCollector{
-		redfishClient: redfishClient,
+		redfishClient: ctx.RedfishClient,
 		collectors:    collectors,
 		redfishUp: prometheus.NewGauge(
 			prometheus.GaugeOpts{
@@ -101,23 +95,6 @@ func (r *RedfishCollector) Collect(ch chan<- prometheus.Metric) {
 	ch <- prometheus.MustNewConstMetric(totalScrapeDurationDesc, prometheus.GaugeValue, time.Since(scrapeTime).Seconds())
 }
 
-func newRedfishClient(host string, username string, password string) (*gofish.APIClient, error) {
-
-	url := fmt.Sprintf("https://%s", host)
-
-	config := gofish.ClientConfig{
-		Endpoint: url,
-		Username: username,
-		Password: password,
-		Insecure: true,
-	}
-	redfishClient, err := gofish.Connect(config)
-	if err != nil {
-		return nil, err
-	}
-	return redfishClient, nil
-}
-
 func parseCommonStatusHealth(status gofishcommon.Health) (float64, bool) {
 	if bytes.Equal([]byte(status), []byte("OK")) {
 		return float64(1), true

collector/system_collector.go

@@ -5,8 +5,8 @@ import (
 	"sync"
 
 	"github.com/apex/log"
+	"github.com/jenningsloy318/redfish_exporter/common"
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/stmcginnis/gofish"
 	"github.com/stmcginnis/gofish/redfish"
 )
 
@@ -33,9 +33,8 @@ var (
 
 // SystemCollector implements the prometheus.Collector.
 type SystemCollector struct {
-	redfishClient *gofish.APIClient
-	metrics       map[string]Metric
-	collectLogs   bool
+	Ctx     *common.CollectionContext
+	metrics map[string]Metric
 	prometheus.Collector
 	collectorScrapeStatus *prometheus.GaugeVec
 	Log                   *log.Entry
@@ -101,11 +100,10 @@ func createSystemMetricMap() map[string]Metric {
 }
 
 // NewSystemCollector returns a collector that collecting memory statistics
-func NewSystemCollector(redfishClient *gofish.APIClient, collectLogs bool, logger *log.Entry) *SystemCollector {
+func NewSystemCollector(ctx *common.CollectionContext, logger *log.Entry) *SystemCollector {
 	return &SystemCollector{
-		redfishClient: redfishClient,
-		metrics:       systemMetrics,
-		collectLogs:   collectLogs,
+		metrics: systemMetrics,
+		Ctx:     ctx,
 		Log: logger.WithFields(log.Fields{
 			"collector": "SystemCollector",
 		}),
@@ -131,9 +129,9 @@ func (s *SystemCollector) Describe(ch chan<- *prometheus.Desc) {
 // Collect implements prometheus.Collector.
 func (s *SystemCollector) Collect(ch chan<- prometheus.Metric) {
 	collectorLogContext := s.Log
-	collectLogs := s.collectLogs
+	collectLogs := s.Ctx.CollectLogs
 	//get service
-	service := s.redfishClient.Service
+	service := s.Ctx.RedfishClient.Service
 
 	// get a list of systems from service
 	if systems, err := service.Systems(); err != nil {
@@ -403,7 +401,7 @@ func (s *SystemCollector) Collect(ch chan<- prometheus.Metric) {
 					wg10.Add(len(logServices))
 
 					for _, logService := range logServices {
-						if err = parseLogService(ch, systemMetrics, SystemSubsystem, SystemID, logService, wg10); err != nil {
+						if err = parseLogService(ch, systemMetrics, s.Ctx, systemLogContext, SystemSubsystem, SystemID, logService, wg10); err != nil {
 							systemLogContext.WithField("operation", "system.LogServices()").WithError(err).Error("error getting log entries from log service")
 						}
 					}