From 7fe29973edb027a48f13b5ca9df87f910adb596c Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Wed, 12 Mar 2025 14:44:36 +0000 Subject: [PATCH 01/11] Only collect last 10 logs This query is extremely slow, so limit to 10 logs. Elasticsearch is better for long term storage. --- collector/common_collector.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/collector/common_collector.go b/collector/common_collector.go index 7d83328..32ff5fb 100644 --- a/collector/common_collector.go +++ b/collector/common_collector.go @@ -5,6 +5,7 @@ import ( "sync" "github.com/prometheus/client_golang/prometheus" + "github.com/stmcginnis/gofish/common" "github.com/stmcginnis/gofish/redfish" ) @@ -51,14 +52,23 @@ func parseLogService(ch chan<- prometheus.Metric, metrics map[string]Metric, sub ch <- prometheus.MustNewConstMetric(metrics[fmt.Sprintf("%s_%s", subsystem, "log_service_health_state")].desc, prometheus.GaugeValue, logServiceHealthStateValue, logServiceLabelValues...) } - logEntries, err := logService.Entries() + logEntries, err := logService.FilteredEntries(common.WithTop(10)) if err != nil { return } wg2 := &sync.WaitGroup{} wg2.Add(len(logEntries)) + processed := make(map[string]bool) for _, logEntry := range logEntries { - go parseLogEntry(ch, metrics[fmt.Sprintf("%s_%s", subsystem, "log_entry_severity_state")].desc, collectorID, logServiceName, logServiceID, logEntry, wg2) + _, exists := processed[logEntry.MessageID] + if exists { + wg2.Done() + continue + } else { + go parseLogEntry(ch, metrics[fmt.Sprintf("%s_%s", subsystem, "log_entry_severity_state")].desc, collectorID, logServiceName, logServiceID, logEntry, wg2) + } + + processed[logEntry.MessageID] = true } return } From 8b3f3692d8f24dd22e0c394e8bf23787778d5e3a Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Fri, 14 Mar 2025 18:58:30 +0000 Subject: [PATCH 02/11] Make logcount a query parameter --- collector/chassis_collector.go | 18 ++++----- collector/common_collector.go | 13 +++++-- collector/manager_collector.go | 18 ++++----- collector/redfish_collector.go | 39 ++++---------------- collector/system_collector.go | 20 +++++----- config.go => common/config.go | 12 +++++- common/context.go | 67 ++++++++++++++++++++++++++++++++++ main.go | 25 +++++-------- 8 files changed, 130 insertions(+), 82 deletions(-) rename config.go => common/config.go (89%) create mode 100644 common/context.go diff --git a/collector/chassis_collector.go b/collector/chassis_collector.go index ad49dc4..531654e 100755 --- a/collector/chassis_collector.go +++ b/collector/chassis_collector.go @@ -7,8 +7,8 @@ import ( "sync" "github.com/apex/log" + "github.com/jenningsloy318/redfish_exporter/common" "github.com/prometheus/client_golang/prometheus" - "github.com/stmcginnis/gofish" "github.com/stmcginnis/gofish/redfish" ) @@ -33,9 +33,8 @@ var ( // ChassisCollector implements the prometheus.Collector. type ChassisCollector struct { - redfishClient *gofish.APIClient + Ctx *common.CollectionContext metrics map[string]Metric - collectLogs bool collectorScrapeStatus *prometheus.GaugeVec Log *log.Entry } @@ -91,13 +90,12 @@ func createChassisMetricMap() map[string]Metric { } // NewChassisCollector returns a collector that collecting chassis statistics -func NewChassisCollector(redfishClient *gofish.APIClient, collectLogs bool, logger *log.Entry) *ChassisCollector { +func NewChassisCollector(ctx *common.CollectionContext, logger *log.Entry) *ChassisCollector { // get service from redfish client return &ChassisCollector{ - redfishClient: redfishClient, - metrics: chassisMetrics, - collectLogs: collectLogs, + metrics: chassisMetrics, + Ctx: ctx, Log: logger.WithFields(log.Fields{ "collector": "ChassisCollector", }), @@ -124,8 +122,8 @@ func (c *ChassisCollector) Describe(ch chan<- *prometheus.Desc) { // Collect implemented prometheus.Collector func (c *ChassisCollector) Collect(ch chan<- prometheus.Metric) { collectorLogContext := c.Log - collectLogs := c.collectLogs - service := c.redfishClient.Service + collectLogs := c.Ctx.CollectLogs + service := c.Ctx.RedfishClient.Service // get a list of chassis from service if chassises, err := service.Chassis(); err != nil { @@ -252,7 +250,7 @@ func (c *ChassisCollector) Collect(ch chan<- prometheus.Metric) { wg6.Add(len(logServices)) for _, logService := range logServices { - if err = parseLogService(ch, chassisMetrics, ChassisSubsystem, chassisID, logService, wg6); err != nil { + if err = parseLogService(ch, chassisMetrics, c.Ctx, ChassisSubsystem, chassisID, logService, wg6); err != nil { chassisLogContext.WithField("operation", "chassis.LogServices()").WithError(err).Error("error getting log entries from log service") } } diff --git a/collector/common_collector.go b/collector/common_collector.go index 32ff5fb..97d1470 100644 --- a/collector/common_collector.go +++ b/collector/common_collector.go @@ -4,6 +4,7 @@ import ( "fmt" "sync" + redfish_common "github.com/jenningsloy318/redfish_exporter/common" "github.com/prometheus/client_golang/prometheus" "github.com/stmcginnis/gofish/common" "github.com/stmcginnis/gofish/redfish" @@ -34,7 +35,7 @@ func addToMetricMap(metricMap map[string]Metric, subsystem, name, help string, v } } -func parseLogService(ch chan<- prometheus.Metric, metrics map[string]Metric, subsystem, collectorID string, logService *redfish.LogService, wg *sync.WaitGroup) (err error) { +func parseLogService(ch chan<- prometheus.Metric, metrics map[string]Metric, ctx *redfish_common.CollectionContext, subsystem, collectorID string, logService *redfish.LogService, wg *sync.WaitGroup) (err error) { defer wg.Done() logServiceName := logService.Name logServiceID := logService.ID @@ -51,8 +52,14 @@ func parseLogService(ch chan<- prometheus.Metric, metrics map[string]Metric, sub if logServiceHealthStateValue, ok := parseCommonStatusHealth(logServiceHealthState); ok { ch <- prometheus.MustNewConstMetric(metrics[fmt.Sprintf("%s_%s", subsystem, "log_service_health_state")].desc, prometheus.GaugeValue, logServiceHealthStateValue, logServiceLabelValues...) } - - logEntries, err := logService.FilteredEntries(common.WithTop(10)) + var ( + logEntries []*redfish.LogEntry + ) + if ctx.LogCount != 0 { + logEntries, err = logService.FilteredEntries(common.WithTop(ctx.LogCount)) + } else { + logEntries, err = logService.Entries() + } if err != nil { return } diff --git a/collector/manager_collector.go b/collector/manager_collector.go index e8591d8..43d8833 100755 --- a/collector/manager_collector.go +++ b/collector/manager_collector.go @@ -5,8 +5,8 @@ import ( "sync" "github.com/apex/log" + "github.com/jenningsloy318/redfish_exporter/common" "github.com/prometheus/client_golang/prometheus" - "github.com/stmcginnis/gofish" ) // ManagerSubmanager is the manager subsystem @@ -22,9 +22,8 @@ var ( // ManagerCollector implements the prometheus.Collector. type ManagerCollector struct { - redfishClient *gofish.APIClient + Ctx *common.CollectionContext metrics map[string]Metric - collectLogs bool collectorScrapeStatus *prometheus.GaugeVec Log *log.Entry } @@ -43,11 +42,10 @@ func createManagerMetricMap() map[string]Metric { } // NewManagerCollector returns a collector that collecting memory statistics -func NewManagerCollector(redfishClient *gofish.APIClient, collectLogs bool, logger *log.Entry) *ManagerCollector { +func NewManagerCollector(ctx *common.CollectionContext, logger *log.Entry) *ManagerCollector { return &ManagerCollector{ - redfishClient: redfishClient, - metrics: managerMetrics, - collectLogs: collectLogs, + metrics: managerMetrics, + Ctx: ctx, Log: logger.WithFields(log.Fields{ "collector": "ManagerCollector", }), @@ -74,9 +72,9 @@ func (m *ManagerCollector) Describe(ch chan<- *prometheus.Desc) { // Collect implemented prometheus.Collector func (m *ManagerCollector) Collect(ch chan<- prometheus.Metric) { collectorLogContext := m.Log - collectLogs := m.collectLogs + collectLogs := m.Ctx.CollectLogs //get service - service := m.redfishClient.Service + service := m.Ctx.RedfishClient.Service // get a list of managers from service if managers, err := service.Managers(); err != nil { @@ -118,7 +116,7 @@ func (m *ManagerCollector) Collect(ch chan<- prometheus.Metric) { wg.Add(len(logServices)) for _, logService := range logServices { - if err = parseLogService(ch, managerMetrics, ManagerSubmanager, ManagerID, logService, wg); err != nil { + if err = parseLogService(ch, managerMetrics, m.Ctx, ManagerSubmanager, ManagerID, logService, wg); err != nil { managerLogContext.WithField("operation", "manager.LogServices()").WithError(err).Error("error getting log entries from log service") } } diff --git a/collector/redfish_collector.go b/collector/redfish_collector.go index 95b1dc5..30da33d 100755 --- a/collector/redfish_collector.go +++ b/collector/redfish_collector.go @@ -2,11 +2,11 @@ package collector import ( "bytes" - "fmt" "sync" "time" - "github.com/apex/log" + alog "github.com/apex/log" + "github.com/jenningsloy318/redfish_exporter/common" "github.com/prometheus/client_golang/prometheus" gofish "github.com/stmcginnis/gofish" gofishcommon "github.com/stmcginnis/gofish/common" @@ -40,22 +40,16 @@ type RedfishCollector struct { } // NewRedfishCollector return RedfishCollector -func NewRedfishCollector(host string, username string, password string, collectLogs bool, logger *log.Entry) *RedfishCollector { +func NewRedfishCollector(ctx *common.CollectionContext, logger *alog.Entry) *RedfishCollector { var collectors map[string]prometheus.Collector - collectorLogCtx := logger - redfishClient, err := newRedfishClient(host, username, password) - if err != nil { - collectorLogCtx.WithError(err).Error("error creating redfish client") - } else { - chassisCollector := NewChassisCollector(redfishClient, collectLogs, collectorLogCtx) - systemCollector := NewSystemCollector(redfishClient, collectLogs, collectorLogCtx) - managerCollector := NewManagerCollector(redfishClient, collectLogs, collectorLogCtx) + chassisCollector := NewChassisCollector(ctx, logger) + systemCollector := NewSystemCollector(ctx, logger) + managerCollector := NewManagerCollector(ctx, logger) - collectors = map[string]prometheus.Collector{"chassis": chassisCollector, "system": systemCollector, "manager": managerCollector} - } + collectors = map[string]prometheus.Collector{"chassis": chassisCollector, "system": systemCollector, "manager": managerCollector} return &RedfishCollector{ - redfishClient: redfishClient, + redfishClient: ctx.RedfishClient, collectors: collectors, redfishUp: prometheus.NewGauge( prometheus.GaugeOpts{ @@ -101,23 +95,6 @@ func (r *RedfishCollector) Collect(ch chan<- prometheus.Metric) { ch <- prometheus.MustNewConstMetric(totalScrapeDurationDesc, prometheus.GaugeValue, time.Since(scrapeTime).Seconds()) } -func newRedfishClient(host string, username string, password string) (*gofish.APIClient, error) { - - url := fmt.Sprintf("https://%s", host) - - config := gofish.ClientConfig{ - Endpoint: url, - Username: username, - Password: password, - Insecure: true, - } - redfishClient, err := gofish.Connect(config) - if err != nil { - return nil, err - } - return redfishClient, nil -} - func parseCommonStatusHealth(status gofishcommon.Health) (float64, bool) { if bytes.Equal([]byte(status), []byte("OK")) { return float64(1), true diff --git a/collector/system_collector.go b/collector/system_collector.go index 9ed8a4f..e45d4f4 100755 --- a/collector/system_collector.go +++ b/collector/system_collector.go @@ -5,8 +5,8 @@ import ( "sync" "github.com/apex/log" + "github.com/jenningsloy318/redfish_exporter/common" "github.com/prometheus/client_golang/prometheus" - "github.com/stmcginnis/gofish" "github.com/stmcginnis/gofish/redfish" ) @@ -33,9 +33,8 @@ var ( // SystemCollector implements the prometheus.Collector. type SystemCollector struct { - redfishClient *gofish.APIClient - metrics map[string]Metric - collectLogs bool + Ctx *common.CollectionContext + metrics map[string]Metric prometheus.Collector collectorScrapeStatus *prometheus.GaugeVec Log *log.Entry @@ -101,11 +100,10 @@ func createSystemMetricMap() map[string]Metric { } // NewSystemCollector returns a collector that collecting memory statistics -func NewSystemCollector(redfishClient *gofish.APIClient, collectLogs bool, logger *log.Entry) *SystemCollector { +func NewSystemCollector(ctx *common.CollectionContext, logger *log.Entry) *SystemCollector { return &SystemCollector{ - redfishClient: redfishClient, - metrics: systemMetrics, - collectLogs: collectLogs, + metrics: systemMetrics, + Ctx: ctx, Log: logger.WithFields(log.Fields{ "collector": "SystemCollector", }), @@ -131,9 +129,9 @@ func (s *SystemCollector) Describe(ch chan<- *prometheus.Desc) { // Collect implements prometheus.Collector. func (s *SystemCollector) Collect(ch chan<- prometheus.Metric) { collectorLogContext := s.Log - collectLogs := s.collectLogs + collectLogs := s.Ctx.CollectLogs //get service - service := s.redfishClient.Service + service := s.Ctx.RedfishClient.Service // get a list of systems from service if systems, err := service.Systems(); err != nil { @@ -379,7 +377,7 @@ func (s *SystemCollector) Collect(ch chan<- prometheus.Metric) { wg10.Add(len(logServices)) for _, logService := range logServices { - if err = parseLogService(ch, systemMetrics, SystemSubsystem, SystemID, logService, wg10); err != nil { + if err = parseLogService(ch, systemMetrics, s.Ctx, SystemSubsystem, SystemID, logService, wg10); err != nil { systemLogContext.WithField("operation", "system.LogServices()").WithError(err).Error("error getting log entries from log service") } } diff --git a/config.go b/common/config.go similarity index 89% rename from config.go rename to common/config.go index 071b722..eceeea8 100755 --- a/config.go +++ b/common/config.go @@ -1,4 +1,4 @@ -package main +package common import ( "fmt" @@ -13,6 +13,7 @@ type Config struct { Groups map[string]HostConfig `yaml:"groups"` Loglevel string `yaml:"loglevel"` Collectlogs *bool `yaml:"collectlogs,omitempty"` + LogCount int `yaml:"logcount,omitempty"` } type SafeConfig struct { @@ -90,3 +91,12 @@ func (sc *SafeConfig) CollectLogs() bool { } return *sc.C.Collectlogs } + +func (sc *SafeConfig) LogCount() int { + sc.Lock() + defer sc.Unlock() + if sc.C.Collectlogs == nil { + return 0 + } + return sc.C.LogCount +} diff --git a/common/context.go b/common/context.go new file mode 100644 index 0000000..b9fb005 --- /dev/null +++ b/common/context.go @@ -0,0 +1,67 @@ +package common + +import ( + "fmt" + "net/http" + "strconv" + + alog "github.com/apex/log" + gofish "github.com/stmcginnis/gofish" +) + +type CollectionContext struct { + Target string + Config *SafeConfig + Request *http.Request + RedfishClient *gofish.APIClient + CollectLogs bool + LogCount int +} + +func NewCollectionContext(target string, config *SafeConfig, r *http.Request, username string, password string, logger *alog.Entry) (*CollectionContext, error) { + client, err := newRedfishClient(target, username, password) + if err != nil { + logger.WithError(err).Error("error creating redfish client") + return nil, err + } + // Support optionally overriding logCounts setting using a query parameter + logCount := config.LogCount() + logCountOverride := r.URL.Query().Get("logcount") + if logCountOverride != "" { + if logCountQuery, err := strconv.Atoi(logCountOverride); err != nil { + logger.WithError(err).Error("error parsing collectlogs query parameter as a boolean") + } else { + logCount = logCountQuery + } + } + logger.WithField("operation", "NewCollectionContext()").Info(fmt.Sprintf("logcount=%d", logCount)) + + // Support optionally overriding collectlogs setting using a query parameter + collectLogs := config.CollectLogs() + collectLogsOverride := r.URL.Query().Get("collectlogs") + if collectLogsOverride != "" { + if collectLogsQuery, err := strconv.ParseBool(collectLogsOverride); err != nil { + logger.WithError(err).Error("error parsing collectlogs query parameter as a boolean") + } else { + collectLogs = collectLogsQuery + } + } + return &CollectionContext{Target: target, Config: config, Request: r, RedfishClient: client, CollectLogs: collectLogs, LogCount: logCount}, nil +} + +func newRedfishClient(host string, username string, password string) (*gofish.APIClient, error) { + + url := fmt.Sprintf("https://%s", host) + + config := gofish.ClientConfig{ + Endpoint: url, + Username: username, + Password: password, + Insecure: true, + } + redfishClient, err := gofish.Connect(config) + if err != nil { + return nil, err + } + return redfishClient, nil +} diff --git a/main.go b/main.go index 2e571ac..45ae57b 100755 --- a/main.go +++ b/main.go @@ -5,12 +5,12 @@ import ( "net/http" "os" "os/signal" - "strconv" "syscall" alog "github.com/apex/log" kitlog "github.com/go-kit/log" "github.com/jenningsloy318/redfish_exporter/collector" + "github.com/jenningsloy318/redfish_exporter/common" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/common/log" @@ -36,8 +36,8 @@ var ( "web.listen-address", "Address to listen on for web interface and telemetry.", ).Default(":9610").String() - sc = &SafeConfig{ - C: &Config{}, + sc = &common.SafeConfig{ + C: &common.Config{}, } reloadCh chan chan error ) @@ -95,7 +95,7 @@ func metricsHandler() http.HandlerFunc { targetLoggerCtx.Info("scraping target host") var ( - hostConfig *HostConfig + hostConfig *common.HostConfig err error ok bool group []string @@ -118,18 +118,12 @@ func metricsHandler() http.HandlerFunc { return } } - - // Support optionally overriding collectlogs setting using a query parameter - collectLogs := sc.CollectLogs() - collectLogsOverride := r.URL.Query().Get("collectlogs") - if collectLogsOverride != "" { - if collectLogs, err = strconv.ParseBool(collectLogsOverride); err != nil { - targetLoggerCtx.WithError(err).Error("error parsing collectlogs query parameter as a boolean") - return - } + collectionCtx, err := common.NewCollectionContext(target, sc, r, hostConfig.Username, hostConfig.Password, targetLoggerCtx) + if err != nil { + targetLoggerCtx.WithError(err).Error("error creating collection context") + return } - - collector := collector.NewRedfishCollector(target, hostConfig.Username, hostConfig.Password, collectLogs, targetLoggerCtx) + collector := collector.NewRedfishCollector(collectionCtx, targetLoggerCtx) registry.MustRegister(collector) gatherers := prometheus.Gatherers{ prometheus.DefaultGatherer, @@ -147,7 +141,6 @@ func main() { kingpin.HelpFlag.Short('h') kingpin.Parse() kitlogger := kitlog.NewLogfmtLogger(os.Stderr) - configLoggerCtx := rootLoggerCtx.WithField("config", *configFile) configLoggerCtx.Info("starting app") // load config first time From d14724c4dd5a33f959a430adc6ebafa7ab3c2c9c Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Fri, 14 Mar 2025 20:03:58 +0000 Subject: [PATCH 03/11] Make log collection options come from group rather than global --- README.md | 16 +++++++++++++- common/config.go | 56 ++++++++++++++++++++++++++++------------------- common/context.go | 13 +++++------ main.go | 2 +- 4 files changed, 56 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index f8f980f..6ada69f 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,21 @@ problematic the collection of logs can be configured as follows: To disable log collection you can set: ```yaml -collectlogs: false +hosts: + default: + username: username + password: password + collectlogs: false +``` + +or for a group: + +```yaml +groups: + group1: + username: username + password: password + collectlogs: false ``` 2) via the `collectlogs` query parameter diff --git a/common/config.go b/common/config.go index eceeea8..052f764 100755 --- a/common/config.go +++ b/common/config.go @@ -9,11 +9,9 @@ import ( ) type Config struct { - Hosts map[string]HostConfig `yaml:"hosts"` - Groups map[string]HostConfig `yaml:"groups"` - Loglevel string `yaml:"loglevel"` - Collectlogs *bool `yaml:"collectlogs,omitempty"` - LogCount int `yaml:"logcount,omitempty"` + Hosts map[string]HostConfigYaml `yaml:"hosts"` + Groups map[string]HostConfigYaml `yaml:"groups"` + Loglevel string `yaml:"loglevel"` } type SafeConfig struct { @@ -21,9 +19,18 @@ type SafeConfig struct { C *Config } +type HostConfigYaml struct { + Username string `yaml:"username"` + Password string `yaml:"password"` + Collectlogs *bool `yaml:"collectlogs,omitempty"` + Logcount *int `yaml:"logcount,omitempty"` +} + type HostConfig struct { - Username string `yaml:"username"` - Password string `yaml:"password"` + Username string + Password string + Collectlogs bool + Logcount int } func (sc *SafeConfig) ReloadConfig(configFile string) error { @@ -49,14 +56,18 @@ func (sc *SafeConfig) HostConfigForTarget(target string) (*HostConfig, error) { defer sc.Unlock() if hostConfig, ok := sc.C.Hosts[target]; ok { return &HostConfig{ - Username: hostConfig.Username, - Password: hostConfig.Password, + Username: hostConfig.Username, + Password: hostConfig.Password, + Collectlogs: hostConfig.CollectLogs(), + Logcount: hostConfig.LogCount(), }, nil } if hostConfig, ok := sc.C.Hosts["default"]; ok { return &HostConfig{ - Username: hostConfig.Username, - Password: hostConfig.Password, + Username: hostConfig.Username, + Password: hostConfig.Password, + Collectlogs: hostConfig.CollectLogs(), + Logcount: hostConfig.LogCount(), }, nil } return &HostConfig{}, fmt.Errorf("no credentials found for target %s", target) @@ -68,7 +79,12 @@ func (sc *SafeConfig) HostConfigForGroup(group string) (*HostConfig, error) { sc.Lock() defer sc.Unlock() if hostConfig, ok := sc.C.Groups[group]; ok { - return &hostConfig, nil + return &HostConfig{ + Username: hostConfig.Username, + Password: hostConfig.Password, + Collectlogs: hostConfig.CollectLogs(), + Logcount: hostConfig.LogCount(), + }, nil } return &HostConfig{}, fmt.Errorf("no credentials found for group %s", group) } @@ -83,20 +99,16 @@ func (sc *SafeConfig) AppLogLevel() string { return "info" } -func (sc *SafeConfig) CollectLogs() bool { - sc.Lock() - defer sc.Unlock() - if sc.C.Collectlogs == nil { +func (hc *HostConfigYaml) CollectLogs() bool { + if hc.Collectlogs == nil { return true } - return *sc.C.Collectlogs + return *hc.Collectlogs } -func (sc *SafeConfig) LogCount() int { - sc.Lock() - defer sc.Unlock() - if sc.C.Collectlogs == nil { +func (hc *HostConfigYaml) LogCount() int { + if hc.Logcount == nil { return 0 } - return sc.C.LogCount + return *hc.Logcount } diff --git a/common/context.go b/common/context.go index b9fb005..c018d54 100644 --- a/common/context.go +++ b/common/context.go @@ -10,22 +10,21 @@ import ( ) type CollectionContext struct { - Target string - Config *SafeConfig Request *http.Request RedfishClient *gofish.APIClient CollectLogs bool LogCount int } -func NewCollectionContext(target string, config *SafeConfig, r *http.Request, username string, password string, logger *alog.Entry) (*CollectionContext, error) { - client, err := newRedfishClient(target, username, password) +func NewCollectionContext(r *http.Request, target string, hostconfig *HostConfig, logger *alog.Entry) (*CollectionContext, error) { + client, err := newRedfishClient(target, hostconfig.Username, hostconfig.Password) if err != nil { logger.WithError(err).Error("error creating redfish client") return nil, err } + // Support optionally overriding logCounts setting using a query parameter - logCount := config.LogCount() + logCount := hostconfig.Logcount logCountOverride := r.URL.Query().Get("logcount") if logCountOverride != "" { if logCountQuery, err := strconv.Atoi(logCountOverride); err != nil { @@ -37,7 +36,7 @@ func NewCollectionContext(target string, config *SafeConfig, r *http.Request, us logger.WithField("operation", "NewCollectionContext()").Info(fmt.Sprintf("logcount=%d", logCount)) // Support optionally overriding collectlogs setting using a query parameter - collectLogs := config.CollectLogs() + collectLogs := hostconfig.Collectlogs collectLogsOverride := r.URL.Query().Get("collectlogs") if collectLogsOverride != "" { if collectLogsQuery, err := strconv.ParseBool(collectLogsOverride); err != nil { @@ -46,7 +45,7 @@ func NewCollectionContext(target string, config *SafeConfig, r *http.Request, us collectLogs = collectLogsQuery } } - return &CollectionContext{Target: target, Config: config, Request: r, RedfishClient: client, CollectLogs: collectLogs, LogCount: logCount}, nil + return &CollectionContext{Request: r, RedfishClient: client, CollectLogs: collectLogs, LogCount: logCount}, nil } func newRedfishClient(host string, username string, password string) (*gofish.APIClient, error) { diff --git a/main.go b/main.go index 45ae57b..2d4eab8 100755 --- a/main.go +++ b/main.go @@ -118,7 +118,7 @@ func metricsHandler() http.HandlerFunc { return } } - collectionCtx, err := common.NewCollectionContext(target, sc, r, hostConfig.Username, hostConfig.Password, targetLoggerCtx) + collectionCtx, err := common.NewCollectionContext(r, target, hostConfig, targetLoggerCtx) if err != nil { targetLoggerCtx.WithError(err).Error("error creating collection context") return From 32f1d142fdb4b281331e87ac5e98a0f71c2c8734 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Fri, 14 Mar 2025 20:13:44 +0000 Subject: [PATCH 04/11] Simplify --- common/config.go | 45 ++++++++++++--------------------------------- 1 file changed, 12 insertions(+), 33 deletions(-) diff --git a/common/config.go b/common/config.go index 052f764..9efedea 100755 --- a/common/config.go +++ b/common/config.go @@ -9,9 +9,9 @@ import ( ) type Config struct { - Hosts map[string]HostConfigYaml `yaml:"hosts"` - Groups map[string]HostConfigYaml `yaml:"groups"` - Loglevel string `yaml:"loglevel"` + Hosts map[string]HostConfig `yaml:"hosts"` + Groups map[string]HostConfig `yaml:"groups"` + Loglevel string `yaml:"loglevel"` } type SafeConfig struct { @@ -19,18 +19,11 @@ type SafeConfig struct { C *Config } -type HostConfigYaml struct { +type HostConfig struct { Username string `yaml:"username"` Password string `yaml:"password"` - Collectlogs *bool `yaml:"collectlogs,omitempty"` - Logcount *int `yaml:"logcount,omitempty"` -} - -type HostConfig struct { - Username string - Password string - Collectlogs bool - Logcount int + Collectlogs bool `yaml:"collectlogs,omitempty"` + Logcount int `yaml:"logcount,omitempty"` } func (sc *SafeConfig) ReloadConfig(configFile string) error { @@ -58,16 +51,16 @@ func (sc *SafeConfig) HostConfigForTarget(target string) (*HostConfig, error) { return &HostConfig{ Username: hostConfig.Username, Password: hostConfig.Password, - Collectlogs: hostConfig.CollectLogs(), - Logcount: hostConfig.LogCount(), + Collectlogs: hostConfig.Collectlogs, + Logcount: hostConfig.Logcount, }, nil } if hostConfig, ok := sc.C.Hosts["default"]; ok { return &HostConfig{ Username: hostConfig.Username, Password: hostConfig.Password, - Collectlogs: hostConfig.CollectLogs(), - Logcount: hostConfig.LogCount(), + Collectlogs: hostConfig.Collectlogs, + Logcount: hostConfig.Logcount, }, nil } return &HostConfig{}, fmt.Errorf("no credentials found for target %s", target) @@ -82,8 +75,8 @@ func (sc *SafeConfig) HostConfigForGroup(group string) (*HostConfig, error) { return &HostConfig{ Username: hostConfig.Username, Password: hostConfig.Password, - Collectlogs: hostConfig.CollectLogs(), - Logcount: hostConfig.LogCount(), + Collectlogs: hostConfig.Collectlogs, + Logcount: hostConfig.Logcount, }, nil } return &HostConfig{}, fmt.Errorf("no credentials found for group %s", group) @@ -98,17 +91,3 @@ func (sc *SafeConfig) AppLogLevel() string { } return "info" } - -func (hc *HostConfigYaml) CollectLogs() bool { - if hc.Collectlogs == nil { - return true - } - return *hc.Collectlogs -} - -func (hc *HostConfigYaml) LogCount() int { - if hc.Logcount == nil { - return 0 - } - return *hc.Logcount -} From 5a3f9d916c399316010145fd4cedcb2f39884608 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Fri, 14 Mar 2025 20:19:01 +0000 Subject: [PATCH 05/11] Reduce noise --- common/context.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/context.go b/common/context.go index c018d54..815e488 100644 --- a/common/context.go +++ b/common/context.go @@ -33,7 +33,7 @@ func NewCollectionContext(r *http.Request, target string, hostconfig *HostConfig logCount = logCountQuery } } - logger.WithField("operation", "NewCollectionContext()").Info(fmt.Sprintf("logcount=%d", logCount)) + //logger.WithField("operation", "NewCollectionContext()").Info(fmt.Sprintf("logcount=%d", logCount)) // Support optionally overriding collectlogs setting using a query parameter collectLogs := hostconfig.Collectlogs From a7d824c7c7a5c9407780aaf62c0c15d5e58458b1 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Mon, 17 Mar 2025 11:25:58 +0000 Subject: [PATCH 06/11] Different log counts per service --- collector/common_collector.go | 9 +++++++-- common/config.go | 8 ++++---- common/context.go | 13 +++---------- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/collector/common_collector.go b/collector/common_collector.go index 97d1470..dc3ed5e 100644 --- a/collector/common_collector.go +++ b/collector/common_collector.go @@ -55,8 +55,13 @@ func parseLogService(ch chan<- prometheus.Metric, metrics map[string]Metric, ctx var ( logEntries []*redfish.LogEntry ) - if ctx.LogCount != 0 { - logEntries, err = logService.FilteredEntries(common.WithTop(ctx.LogCount)) + logCount, ok := ctx.LogCount[logServiceName] + if !ok { + logCount = -1 + } + + if logCount > 0 { + logEntries, err = logService.FilteredEntries(common.WithTop(logCount)) } else { logEntries, err = logService.Entries() } diff --git a/common/config.go b/common/config.go index 9efedea..1b18fd2 100755 --- a/common/config.go +++ b/common/config.go @@ -20,10 +20,10 @@ type SafeConfig struct { } type HostConfig struct { - Username string `yaml:"username"` - Password string `yaml:"password"` - Collectlogs bool `yaml:"collectlogs,omitempty"` - Logcount int `yaml:"logcount,omitempty"` + Username string `yaml:"username"` + Password string `yaml:"password"` + Collectlogs bool `yaml:"collectlogs,omitempty"` + Logcount map[string]int `yaml:"logcount,omitempty"` } func (sc *SafeConfig) ReloadConfig(configFile string) error { diff --git a/common/context.go b/common/context.go index 815e488..dee8732 100644 --- a/common/context.go +++ b/common/context.go @@ -13,7 +13,7 @@ type CollectionContext struct { Request *http.Request RedfishClient *gofish.APIClient CollectLogs bool - LogCount int + LogCount map[string]int } func NewCollectionContext(r *http.Request, target string, hostconfig *HostConfig, logger *alog.Entry) (*CollectionContext, error) { @@ -25,15 +25,8 @@ func NewCollectionContext(r *http.Request, target string, hostconfig *HostConfig // Support optionally overriding logCounts setting using a query parameter logCount := hostconfig.Logcount - logCountOverride := r.URL.Query().Get("logcount") - if logCountOverride != "" { - if logCountQuery, err := strconv.Atoi(logCountOverride); err != nil { - logger.WithError(err).Error("error parsing collectlogs query parameter as a boolean") - } else { - logCount = logCountQuery - } - } - //logger.WithField("operation", "NewCollectionContext()").Info(fmt.Sprintf("logcount=%d", logCount)) + // TODO.. query parameter could logcount_=10 + logger.WithField("operation", "NewCollectionContext()").Info(fmt.Sprintf("logcount=%s", fmt.Sprint(logCount))) // Support optionally overriding collectlogs setting using a query parameter collectLogs := hostconfig.Collectlogs From 86dfc685707ec75f5bed0d91ded1496eb6fbbffd Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Mon, 17 Mar 2025 11:33:52 +0000 Subject: [PATCH 07/11] Switch to ID as it is likely to no contain spaces --- collector/common_collector.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/collector/common_collector.go b/collector/common_collector.go index dc3ed5e..3c4037d 100644 --- a/collector/common_collector.go +++ b/collector/common_collector.go @@ -55,7 +55,7 @@ func parseLogService(ch chan<- prometheus.Metric, metrics map[string]Metric, ctx var ( logEntries []*redfish.LogEntry ) - logCount, ok := ctx.LogCount[logServiceName] + logCount, ok := ctx.LogCount[logServiceID] if !ok { logCount = -1 } From 5afb701a85d24efdbfe3203204bff3dfdf69a58c Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Mon, 17 Mar 2025 11:45:49 +0000 Subject: [PATCH 08/11] Add further example --- README.md | 41 +++++++++++++++++++++++++++++++++++ collector/common_collector.go | 5 ++++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6ada69f..020b008 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,47 @@ curl '127.0.0.1:9123/redfish?target=10.10.12.23&collectlogs=true' The `collectlogs` query parameter can be included in Prometheus config. + +### Log collection count + +To restrict the number of logs collected for all log services: + +``` +hosts: + default: + username: username + password: password + collectlogs: true + logcount: + DEFAULT: 10 +``` + +For a particular log service, `Sel`: + +``` +hosts: + default: + username: username + password: password + collectlogs: true + logcount: + Sel: 20 +``` + +Collect all logs for one service, `Sel` , but limit others: + +``` +hosts: + default: + username: username + password: password + collectlogs: true + logcount: + DEFAULT: 10 + Sel: -1 +``` + + ## Building To build the redfish_exporter executable run the command: diff --git a/collector/common_collector.go b/collector/common_collector.go index 3c4037d..32b89c7 100644 --- a/collector/common_collector.go +++ b/collector/common_collector.go @@ -57,7 +57,10 @@ func parseLogService(ch chan<- prometheus.Metric, metrics map[string]Metric, ctx ) logCount, ok := ctx.LogCount[logServiceID] if !ok { - logCount = -1 + logCount, ok = ctx.LogCount["DEFAULT"] + if !ok { + logCount = -1 + } } if logCount > 0 { From a7cfcc43ddf414e6d67592d19a2b4c2ff7bca679 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Mon, 17 Mar 2025 14:47:29 +0000 Subject: [PATCH 09/11] Log after lookup --- collector/chassis_collector.go | 2 +- collector/common_collector.go | 5 ++++- collector/manager_collector.go | 2 +- collector/system_collector.go | 2 +- common/context.go | 2 -- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/collector/chassis_collector.go b/collector/chassis_collector.go index 531654e..cb6f633 100755 --- a/collector/chassis_collector.go +++ b/collector/chassis_collector.go @@ -250,7 +250,7 @@ func (c *ChassisCollector) Collect(ch chan<- prometheus.Metric) { wg6.Add(len(logServices)) for _, logService := range logServices { - if err = parseLogService(ch, chassisMetrics, c.Ctx, ChassisSubsystem, chassisID, logService, wg6); err != nil { + if err = parseLogService(ch, chassisMetrics, c.Ctx, chassisLogContext, ChassisSubsystem, chassisID, logService, wg6); err != nil { chassisLogContext.WithField("operation", "chassis.LogServices()").WithError(err).Error("error getting log entries from log service") } } diff --git a/collector/common_collector.go b/collector/common_collector.go index 32b89c7..2a1c1b0 100644 --- a/collector/common_collector.go +++ b/collector/common_collector.go @@ -4,6 +4,7 @@ import ( "fmt" "sync" + "github.com/apex/log" redfish_common "github.com/jenningsloy318/redfish_exporter/common" "github.com/prometheus/client_golang/prometheus" "github.com/stmcginnis/gofish/common" @@ -35,7 +36,7 @@ func addToMetricMap(metricMap map[string]Metric, subsystem, name, help string, v } } -func parseLogService(ch chan<- prometheus.Metric, metrics map[string]Metric, ctx *redfish_common.CollectionContext, subsystem, collectorID string, logService *redfish.LogService, wg *sync.WaitGroup) (err error) { +func parseLogService(ch chan<- prometheus.Metric, metrics map[string]Metric, ctx *redfish_common.CollectionContext, logger *log.Entry, subsystem, collectorID string, logService *redfish.LogService, wg *sync.WaitGroup) (err error) { defer wg.Done() logServiceName := logService.Name logServiceID := logService.ID @@ -63,6 +64,8 @@ func parseLogService(ch chan<- prometheus.Metric, metrics map[string]Metric, ctx } } + logger.WithField("operation", "parseLogService").Info(fmt.Sprintf("logcount=%d", logCount)) + if logCount > 0 { logEntries, err = logService.FilteredEntries(common.WithTop(logCount)) } else { diff --git a/collector/manager_collector.go b/collector/manager_collector.go index 43d8833..7e40fc7 100755 --- a/collector/manager_collector.go +++ b/collector/manager_collector.go @@ -116,7 +116,7 @@ func (m *ManagerCollector) Collect(ch chan<- prometheus.Metric) { wg.Add(len(logServices)) for _, logService := range logServices { - if err = parseLogService(ch, managerMetrics, m.Ctx, ManagerSubmanager, ManagerID, logService, wg); err != nil { + if err = parseLogService(ch, managerMetrics, m.Ctx, managerLogContext, ManagerSubmanager, ManagerID, logService, wg); err != nil { managerLogContext.WithField("operation", "manager.LogServices()").WithError(err).Error("error getting log entries from log service") } } diff --git a/collector/system_collector.go b/collector/system_collector.go index e45d4f4..dd869ee 100755 --- a/collector/system_collector.go +++ b/collector/system_collector.go @@ -377,7 +377,7 @@ func (s *SystemCollector) Collect(ch chan<- prometheus.Metric) { wg10.Add(len(logServices)) for _, logService := range logServices { - if err = parseLogService(ch, systemMetrics, s.Ctx, SystemSubsystem, SystemID, logService, wg10); err != nil { + if err = parseLogService(ch, systemMetrics, s.Ctx, systemLogContext, SystemSubsystem, SystemID, logService, wg10); err != nil { systemLogContext.WithField("operation", "system.LogServices()").WithError(err).Error("error getting log entries from log service") } } diff --git a/common/context.go b/common/context.go index dee8732..2e28456 100644 --- a/common/context.go +++ b/common/context.go @@ -23,10 +23,8 @@ func NewCollectionContext(r *http.Request, target string, hostconfig *HostConfig return nil, err } - // Support optionally overriding logCounts setting using a query parameter logCount := hostconfig.Logcount // TODO.. query parameter could logcount_=10 - logger.WithField("operation", "NewCollectionContext()").Info(fmt.Sprintf("logcount=%s", fmt.Sprint(logCount))) // Support optionally overriding collectlogs setting using a query parameter collectLogs := hostconfig.Collectlogs From 970ef9308c8240d999bb01c62af8523e22f45adb Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Mon, 17 Mar 2025 14:50:06 +0000 Subject: [PATCH 10/11] Add more context to log message --- collector/common_collector.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/collector/common_collector.go b/collector/common_collector.go index 2a1c1b0..47c4f21 100644 --- a/collector/common_collector.go +++ b/collector/common_collector.go @@ -64,7 +64,7 @@ func parseLogService(ch chan<- prometheus.Metric, metrics map[string]Metric, ctx } } - logger.WithField("operation", "parseLogService").Info(fmt.Sprintf("logcount=%d", logCount)) + logger.WithField("operation", "parseLogService").Info(fmt.Sprintf("logServiceID=%s, logcount=%d", logServiceID, logCount)) if logCount > 0 { logEntries, err = logService.FilteredEntries(common.WithTop(logCount)) From 7400129c367111f296077184527ed568ffcc8477 Mon Sep 17 00:00:00 2001 From: Doug Szumski Date: Wed, 17 Sep 2025 11:22:39 +0100 Subject: [PATCH 11/11] Update common_collector.go --- collector/common_collector.go | 1 + 1 file changed, 1 insertion(+) diff --git a/collector/common_collector.go b/collector/common_collector.go index bb1b947..7f56383 100644 --- a/collector/common_collector.go +++ b/collector/common_collector.go @@ -82,6 +82,7 @@ func parseLogService(ch chan<- prometheus.Metric, metrics map[string]Metric, ctx if exists { wg2.Done() continue + } go parseLogEntry(ch, metrics[fmt.Sprintf("%s_%s", subsystem, "log_entry_severity_state")].desc, collectorID, logServiceName, logServiceID, logEntry, wg2) processed[logEntry.MessageID] = true }