Stackhpc upstream master (#5)

* Add config option to not collect logs

Collecting logs from BMCs can be very slow, so you may wish to
disable doing so. However, the logs can still be useful.

This change adds support for optionally overriding the `collectlogs`
config setting. One use case envisaged would be to set collectlogs
to false in the config, allowing the BMC to be scraped for data
every few minutes. Then every hour or so, the logs could be collected
via the `collectlogs` override.

---------

Co-authored-by: Matt Anson <[email protected]>

Author: dougszumski

README.md

@@ -20,6 +20,46 @@ groups:
 Note that the ```default``` entry is useful as it avoids an error
 condition that is discussed in [this issue][2].
 
+## Log collection
+
+One issue you may encounter is slow scrape times when collecting logs from
+BMCs. Sometimes these scrapes can take ~10minutes. If this becomes
+problematic the collection of logs can be configured as follows:
+
+1) via the the config file
+
+To disable log collection you can set:
+
+```yaml
+collectlogs: false
+```
+
+2) via the `collectlogs` query parameter
+
+For example:
+
+```sh
+curl '127.0.0.1:9123/redfish?target=10.10.12.23&collectlogs=false'
+```
+
+3) Via a combination of both options
+
+For example, collect metrics without logs by default:
+
+Set:
+
+```yaml
+collectlogs: false
+```
+
+Every hour or so, override the config setting and fetch the logs:
+
+```sh
+curl '127.0.0.1:9123/redfish?target=10.10.12.23&collectlogs=true'
+```
+
+The `collectlogs` query parameter can be included in Prometheus config.
+
 ## Building
 
 To build the redfish_exporter executable run the command:

collector/chassis_collector.go

@@ -35,6 +35,7 @@ var (
 type ChassisCollector struct {
 	redfishClient         *gofish.APIClient
 	metrics               map[string]Metric
+	collectLogs           bool
 	collectorScrapeStatus *prometheus.GaugeVec
 	Log                   *log.Entry
 }
@@ -90,12 +91,13 @@ func createChassisMetricMap() map[string]Metric {
 }
 
 // NewChassisCollector returns a collector that collecting chassis statistics
-func NewChassisCollector(redfishClient *gofish.APIClient, logger *log.Entry) *ChassisCollector {
+func NewChassisCollector(redfishClient *gofish.APIClient, collectLogs bool, logger *log.Entry) *ChassisCollector {
 	// get service from redfish client
 
 	return &ChassisCollector{
 		redfishClient: redfishClient,
 		metrics:       chassisMetrics,
+		collectLogs:   collectLogs,
 		Log: logger.WithFields(log.Fields{
 			"collector": "ChassisCollector",
 		}),
@@ -122,6 +124,7 @@ func (c *ChassisCollector) Describe(ch chan<- *prometheus.Desc) {
 // Collect implemented prometheus.Collector
 func (c *ChassisCollector) Collect(ch chan<- prometheus.Metric) {
 	collectorLogContext := c.Log
+	collectLogs := c.collectLogs
 	service := c.redfishClient.Service
 
 	// get a list of chassis from service
@@ -238,18 +241,20 @@ func (c *ChassisCollector) Collect(ch chan<- prometheus.Metric) {
 			}
 
 			// process log services
-			logServices, err := chassis.LogServices()
-			if err != nil {
-				chassisLogContext.WithField("operation", "chassis.LogServices()").WithError(err).Error("error getting log services from chassis")
-			} else if logServices == nil {
-				chassisLogContext.WithField("operation", "chassis.LogServices()").Info("no log services found")
-			} else {
-				wg6 := &sync.WaitGroup{}
-				wg6.Add(len(logServices))
-
-				for _, logService := range logServices {
-					if err = parseLogService(ch, chassisMetrics, ChassisSubsystem, chassisID, logService, wg6); err != nil {
-						chassisLogContext.WithField("operation", "chassis.LogServices()").WithError(err).Error("error getting log entries from log service")
+			if collectLogs {
+				logServices, err := chassis.LogServices()
+				if err != nil {
+					chassisLogContext.WithField("operation", "chassis.LogServices()").WithError(err).Error("error getting log services from chassis")
+				} else if logServices == nil {
+					chassisLogContext.WithField("operation", "chassis.LogServices()").Info("no log services found")
+				} else {
+					wg6 := &sync.WaitGroup{}
+					wg6.Add(len(logServices))
+
+					for _, logService := range logServices {
+						if err = parseLogService(ch, chassisMetrics, ChassisSubsystem, chassisID, logService, wg6); err != nil {
+							chassisLogContext.WithField("operation", "chassis.LogServices()").WithError(err).Error("error getting log entries from log service")
+						}
 					}
 				}
 			}

collector/manager_collector.go

@@ -24,6 +24,7 @@ var (
 type ManagerCollector struct {
 	redfishClient         *gofish.APIClient
 	metrics               map[string]Metric
+	collectLogs           bool
 	collectorScrapeStatus *prometheus.GaugeVec
 	Log                   *log.Entry
 }
@@ -42,10 +43,11 @@ func createManagerMetricMap() map[string]Metric {
 }
 
 // NewManagerCollector returns a collector that collecting memory statistics
-func NewManagerCollector(redfishClient *gofish.APIClient, logger *log.Entry) *ManagerCollector {
+func NewManagerCollector(redfishClient *gofish.APIClient, collectLogs bool, logger *log.Entry) *ManagerCollector {
 	return &ManagerCollector{
 		redfishClient: redfishClient,
 		metrics:       managerMetrics,
+		collectLogs:   collectLogs,
 		Log: logger.WithFields(log.Fields{
 			"collector": "ManagerCollector",
 		}),
@@ -72,6 +74,7 @@ func (m *ManagerCollector) Describe(ch chan<- *prometheus.Desc) {
 // Collect implemented prometheus.Collector
 func (m *ManagerCollector) Collect(ch chan<- prometheus.Metric) {
 	collectorLogContext := m.Log
+	collectLogs := m.collectLogs
 	//get service
 	service := m.redfishClient.Service
 
@@ -104,18 +107,20 @@ func (m *ManagerCollector) Collect(ch chan<- prometheus.Metric) {
 			}
 
 			// process log services
-			logServices, err := manager.LogServices()
-			if err != nil {
-				managerLogContext.WithField("operation", "manager.LogServices()").WithError(err).Error("error getting log services from manager")
-			} else if logServices == nil {
-				managerLogContext.WithField("operation", "manager.LogServices()").Info("no log services found")
-			} else {
-				wg := &sync.WaitGroup{}
-				wg.Add(len(logServices))
-
-				for _, logService := range logServices {
-					if err = parseLogService(ch, managerMetrics, ManagerSubmanager, ManagerID, logService, wg); err != nil {
-						managerLogContext.WithField("operation", "manager.LogServices()").WithError(err).Error("error getting log entries from log service")
+			if collectLogs {
+				logServices, err := manager.LogServices()
+				if err != nil {
+					managerLogContext.WithField("operation", "manager.LogServices()").WithError(err).Error("error getting log services from manager")
+				} else if logServices == nil {
+					managerLogContext.WithField("operation", "manager.LogServices()").Info("no log services found")
+				} else {
+					wg := &sync.WaitGroup{}
+					wg.Add(len(logServices))
+
+					for _, logService := range logServices {
+						if err = parseLogService(ch, managerMetrics, ManagerSubmanager, ManagerID, logService, wg); err != nil {
+							managerLogContext.WithField("operation", "manager.LogServices()").WithError(err).Error("error getting log entries from log service")
+						}
 					}
 				}
 			}

collector/redfish_collector.go

@@ -40,16 +40,16 @@ type RedfishCollector struct {
 }
 
 // NewRedfishCollector return RedfishCollector
-func NewRedfishCollector(host string, username string, password string, logger *log.Entry) *RedfishCollector {
+func NewRedfishCollector(host string, username string, password string, collectLogs bool, logger *log.Entry) *RedfishCollector {
 	var collectors map[string]prometheus.Collector
 	collectorLogCtx := logger
 	redfishClient, err := newRedfishClient(host, username, password)
 	if err != nil {
 		collectorLogCtx.WithError(err).Error("error creating redfish client")
 	} else {
-		chassisCollector := NewChassisCollector(redfishClient, collectorLogCtx)
-		systemCollector := NewSystemCollector(redfishClient, collectorLogCtx)
-		managerCollector := NewManagerCollector(redfishClient, collectorLogCtx)
+		chassisCollector := NewChassisCollector(redfishClient, collectLogs, collectorLogCtx)
+		systemCollector := NewSystemCollector(redfishClient, collectLogs, collectorLogCtx)
+		managerCollector := NewManagerCollector(redfishClient, collectLogs, collectorLogCtx)
 
 		collectors = map[string]prometheus.Collector{"chassis": chassisCollector, "system": systemCollector, "manager": managerCollector}
 	}

collector/system_collector.go

@@ -35,6 +35,7 @@ var (
 type SystemCollector struct {
 	redfishClient *gofish.APIClient
 	metrics       map[string]Metric
+	collectLogs   bool
 	prometheus.Collector
 	collectorScrapeStatus *prometheus.GaugeVec
 	Log                   *log.Entry
@@ -100,10 +101,11 @@ func createSystemMetricMap() map[string]Metric {
 }
 
 // NewSystemCollector returns a collector that collecting memory statistics
-func NewSystemCollector(redfishClient *gofish.APIClient, logger *log.Entry) *SystemCollector {
+func NewSystemCollector(redfishClient *gofish.APIClient, collectLogs bool, logger *log.Entry) *SystemCollector {
 	return &SystemCollector{
 		redfishClient: redfishClient,
 		metrics:       systemMetrics,
+		collectLogs:   collectLogs,
 		Log: logger.WithFields(log.Fields{
 			"collector": "SystemCollector",
 		}),
@@ -129,6 +131,7 @@ func (s *SystemCollector) Describe(ch chan<- *prometheus.Desc) {
 // Collect implements prometheus.Collector.
 func (s *SystemCollector) Collect(ch chan<- prometheus.Metric) {
 	collectorLogContext := s.Log
+	collectLogs := s.collectLogs
 	//get service
 	service := s.redfishClient.Service
 
@@ -366,17 +369,19 @@ func (s *SystemCollector) Collect(ch chan<- prometheus.Metric) {
 			}
 
 			// process log services
-			logServices, err := system.LogServices()
-			if err != nil {
-				systemLogContext.WithField("operation", "system.LogServices()").WithError(err).Error("error getting log services from system")
-			} else if logServices == nil {
-				systemLogContext.WithField("operation", "system.LogServices()").Info("no log services found")
-			} else {
-				wg10.Add(len(logServices))
-
-				for _, logService := range logServices {
-					if err = parseLogService(ch, systemMetrics, SystemSubsystem, SystemID, logService, wg10); err != nil {
-						systemLogContext.WithField("operation", "system.LogServices()").WithError(err).Error("error getting log entries from log service")
+			if collectLogs {
+				logServices, err := system.LogServices()
+				if err != nil {
+					systemLogContext.WithField("operation", "system.LogServices()").WithError(err).Error("error getting log services from system")
+				} else if logServices == nil {
+					systemLogContext.WithField("operation", "system.LogServices()").Info("no log services found")
+				} else {
+					wg10.Add(len(logServices))
+
+					for _, logService := range logServices {
+						if err = parseLogService(ch, systemMetrics, SystemSubsystem, SystemID, logService, wg10); err != nil {
+							systemLogContext.WithField("operation", "system.LogServices()").WithError(err).Error("error getting log entries from log service")
+						}
 					}
 				}
 			}

config.go

@@ -9,9 +9,10 @@ import (
 )
 
 type Config struct {
-	Hosts    map[string]HostConfig `yaml:"hosts"`
-	Groups   map[string]HostConfig `yaml:"groups"`
-	Loglevel string                `yaml:"loglevel"`
+	Hosts       map[string]HostConfig `yaml:"hosts"`
+	Groups      map[string]HostConfig `yaml:"groups"`
+	Loglevel    string                `yaml:"loglevel"`
+	Collectlogs *bool                 `yaml:"collectlogs,omitempty"`
 }
 
 type SafeConfig struct {
@@ -80,3 +81,12 @@ func (sc *SafeConfig) AppLogLevel() string {
 	}
 	return "info"
 }
+
+func (sc *SafeConfig) CollectLogs() bool {
+	sc.Lock()
+	defer sc.Unlock()
+	if sc.C.Collectlogs == nil {
+		return true
+	}
+	return *sc.C.Collectlogs
+}

config.yml.example

@@ -11,3 +11,4 @@ groups:
     password: group1_pass
 # loglevel can be one of "debug", "info", "warn", "error", or "fatal"
 # loglevel: info
+collectlogs: true

main.go

@@ -5,6 +5,7 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
+	"strconv"
 	"syscall"
 
 	alog "github.com/apex/log"
@@ -118,7 +119,17 @@ func metricsHandler() http.HandlerFunc {
 			}
 		}
 
-		collector := collector.NewRedfishCollector(target, hostConfig.Username, hostConfig.Password, targetLoggerCtx)
+		// Support optionally overriding collectlogs setting using a query parameter
+		collectLogs := sc.CollectLogs()
+		collectLogsOverride := r.URL.Query().Get("collectlogs")
+		if collectLogsOverride != "" {
+			if collectLogs, err = strconv.ParseBool(collectLogsOverride); err != nil {
+				targetLoggerCtx.WithError(err).Error("error parsing collectlogs query parameter as a boolean")
+				return
+			}
+		}
+
+		collector := collector.NewRedfishCollector(target, hostConfig.Username, hostConfig.Password, collectLogs, targetLoggerCtx)
 		registry.MustRegister(collector)
 		gatherers := prometheus.Gatherers{
 			prometheus.DefaultGatherer,
@@ -189,6 +200,7 @@ func main() {
             <h1>redfish Exporter</h1>
             <form action="/redfish">
             <label>Target:</label> <input type="text" name="target" placeholder="X.X.X.X" value="1.2.3.4"><br>
+            <label>CollectLogs:</label> <input type="text" name="collectlogs" placeholder="collectlogs (optional)" value="bool"><br>
             <label>Group:</label> <input type="text" name="group" placeholder="group (optional)" value=""><br>
             <input type="submit" value="Submit">
 						</form>