pkg/services/otelhealth: add otelhealth; update promhealth

Author: jmank88

pkg/loop/server.go

@@ -15,6 +15,7 @@ import (
 	"github.com/smartcontractkit/chainlink-common/pkg/config/build"
 	"github.com/smartcontractkit/chainlink-common/pkg/logger"
 	"github.com/smartcontractkit/chainlink-common/pkg/services"
+	"github.com/smartcontractkit/chainlink-common/pkg/services/otelhealth"
 	"github.com/smartcontractkit/chainlink-common/pkg/settings/limits"
 	"github.com/smartcontractkit/chainlink-common/pkg/services/promhealth"
 	"github.com/smartcontractkit/chainlink-common/pkg/sqlutil"
@@ -154,7 +155,17 @@ func (s *Server) start() error {
 		return fmt.Errorf("error starting prometheus server: %w", err)
 	}
 
-	s.checker = promhealth.NewChecker("", "")
+	var healthCfg services.HealthCheckerConfig
+	healthCfg = promhealth.ConfigureHooks(healthCfg)
+	if bc := beholder.GetClient(); bc != nil {
+		var err error
+		healthCfg, err = otelhealth.ConfigureHooks(healthCfg, bc.Meter)
+		if err != nil {
+			return fmt.Errorf("failed to configure health checker otel hooks: %w", err)
+		}
+	}
+	s.checker = healthCfg.New()
+
 	if err := s.checker.Start(); err != nil {
 		return fmt.Errorf("error starting health checker: %w", err)
 	}

pkg/services/health.go

@@ -1,6 +1,7 @@
 package services
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"maps"
@@ -88,10 +89,10 @@ type HealthCheckerConfig struct {
 	// Optionally override debug.BuildInfo
 	Ver, Sha string
 	// Optional hooks for reporting.
-	IncVersion func(ver string, sha string)
-	AddUptime  func(duration time.Duration)
-	SetStatus  func(name string, status int)
-	Delete     func(name string)
+	IncVersion func(ctx context.Context, ver string, sha string)
+	AddUptime  func(ctx context.Context, duration time.Duration)
+	SetStatus  func(ctx context.Context, name string, status int)
+	Delete     func(ctx context.Context, name string)
 }
 
 func (cfg HealthCheckerConfig) initVerSha() {
@@ -112,16 +113,16 @@ func (cfg HealthCheckerConfig) initVerSha() {
 
 func (cfg HealthCheckerConfig) setNoopHooks() {
 	if cfg.IncVersion == nil {
-		cfg.IncVersion = func(ver, sha string) {}
+		cfg.IncVersion = func(ctx context.Context, ver, sha string) {}
 	}
 	if cfg.AddUptime == nil {
-		cfg.AddUptime = func(d time.Duration) {}
+		cfg.AddUptime = func(ctx context.Context, d time.Duration) {}
 	}
 	if cfg.SetStatus == nil {
-		cfg.SetStatus = func(name string, status int) {}
+		cfg.SetStatus = func(ctx context.Context, name string, status int) {}
 	}
 	if cfg.Delete == nil {
-		cfg.Delete = func(name string) {}
+		cfg.Delete = func(ctx context.Context, name string) {}
 	}
 }
 
@@ -140,10 +141,11 @@ func (cfg HealthCheckerConfig) New() *HealthChecker {
 
 func (c *HealthChecker) Start() error {
 	return c.StartOnce("HealthCheck", func() error {
-		c.cfg.IncVersion(c.cfg.Ver, c.cfg.Sha)
+		ctx := context.Background()
+		c.cfg.IncVersion(ctx, c.cfg.Ver, c.cfg.Sha)
 
 		// update immediately
-		c.update()
+		c.update(ctx)
 
 		go c.run()
 
@@ -162,19 +164,20 @@ func (c *HealthChecker) Close() error {
 func (c *HealthChecker) run() {
 	defer close(c.chDone)
 
+	ctx := context.Background()
 	ticker := time.NewTicker(interval)
 
 	for {
 		select {
 		case <-ticker.C:
-			c.update()
+			c.update(ctx)
 		case <-c.chStop:
 			return
 		}
 	}
 }
 
-func (c *HealthChecker) update() {
+func (c *HealthChecker) update(ctx context.Context) {
 	// copy services into a new map to avoid lock contention while doing checks
 	c.servicesMu.RLock()
 	l := len(c.services)
@@ -196,10 +199,10 @@ func (c *HealthChecker) update() {
 			}
 
 			// report metrics to prometheus
-			c.cfg.SetStatus(name, value)
+			c.cfg.SetStatus(ctx, name, value)
 		}
 	}
-	c.cfg.AddUptime(interval)
+	c.cfg.AddUptime(ctx, interval)
 
 	// save state
 	c.stateMu.Lock()
@@ -231,11 +234,12 @@ func (c *HealthChecker) Unregister(name string) error {
 	if name == "" {
 		return fmt.Errorf("name cannot be empty")
 	}
+	ctx := context.Background()
 
 	c.servicesMu.Lock()
 	defer c.servicesMu.Unlock()
 	delete(c.services, name)
-	c.cfg.Delete(name)
+	c.cfg.Delete(ctx, name)
 	return nil
 }
 

pkg/services/otelhealth/otelhealth.go

@@ -0,0 +1,54 @@
+package otelhealth
+
+import (
+	"context"
+	"time"
+
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
+
+	"github.com/smartcontractkit/chainlink-common/pkg/services"
+)
+
+func NewChecker(ver, sha string, meter metric.Meter) (*services.HealthChecker, error) {
+	cfg, err := ConfigureHooks(services.HealthCheckerConfig{Ver: ver, Sha: sha}, meter)
+	if err != nil {
+		return nil, err
+	}
+	return cfg.New(), nil
+}
+
+func ConfigureHooks(orig services.HealthCheckerConfig, meter metric.Meter) (services.HealthCheckerConfig, error) {
+	cfg := orig // copy
+	healthStatus, err := meter.Int64Gauge("health", metric.WithDescription("Health status by service"))
+	if err != nil {
+		return services.HealthCheckerConfig{}, err
+	}
+	version, err := meter.Int64Counter("version", metric.WithDescription("Version and SHA of the service"))
+	if err != nil {
+		return services.HealthCheckerConfig{}, err
+	}
+	uptimeSeconds, err := meter.Float64Gauge("uptime_seconds", metric.WithDescription("Uptime of the service"))
+	if err != nil {
+		return services.HealthCheckerConfig{}, err
+	}
+	cfg.AddUptime = func(ctx context.Context, d time.Duration) {
+		if orig.AddUptime != nil {
+			orig.AddUptime(ctx, d)
+		}
+		uptimeSeconds.Record(ctx, d.Seconds())
+	}
+	cfg.IncVersion = func(ctx context.Context, ver string, sha string) {
+		if orig.IncVersion != nil {
+			orig.IncVersion(ctx, ver, sha)
+		}
+		version.Add(ctx, 1, metric.WithAttributes(attribute.String("version", ver), attribute.String("commit", sha)))
+	}
+	cfg.SetStatus = func(ctx context.Context, name string, value int) {
+		if orig.SetStatus != nil {
+			orig.SetStatus(ctx, name, value)
+		}
+		healthStatus.Record(ctx, int64(value), metric.WithAttributes(attribute.String("service_id", name)))
+	}
+	return cfg, nil
+}

pkg/services/promhealth/promhealth.go

@@ -1,6 +1,7 @@
 package promhealth
 
 import (
+	"context"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
@@ -34,20 +35,34 @@ var (
 
 // NewChecker returns a *services.HealthChecker with hooks for prometheus metrics.
 func NewChecker(ver, sha string) *services.HealthChecker {
-	return services.HealthCheckerConfig{
-		Ver: ver,
-		Sha: sha,
-		AddUptime: func(d time.Duration) {
-			uptimeSeconds.Add(d.Seconds())
-		},
-		IncVersion: func(ver string, sha string) {
-			version.WithLabelValues(ver, sha).Inc()
-		},
-		SetStatus: func(name string, value int) {
-			healthStatus.WithLabelValues(name).Set(float64(value))
-		},
-		Delete: func(name string) {
-			healthStatus.DeleteLabelValues(name)
-		},
-	}.New()
+	return ConfigureHooks(services.HealthCheckerConfig{Ver: ver, Sha: sha}).New()
+}
+
+func ConfigureHooks(orig services.HealthCheckerConfig) services.HealthCheckerConfig {
+	cfg := orig // copy
+	cfg.AddUptime = func(ctx context.Context, d time.Duration) {
+		if orig.AddUptime != nil {
+			orig.AddUptime(ctx, d)
+		}
+		uptimeSeconds.Add(d.Seconds())
+	}
+	cfg.IncVersion = func(ctx context.Context, ver string, sha string) {
+		if orig.IncVersion != nil {
+			orig.IncVersion(ctx, ver, sha)
+		}
+		version.WithLabelValues(ver, sha).Inc()
+	}
+	cfg.SetStatus = func(ctx context.Context, name string, value int) {
+		if orig.SetStatus != nil {
+			orig.SetStatus(ctx, name, value)
+		}
+		healthStatus.WithLabelValues(name).Set(float64(value))
+	}
+	cfg.Delete = func(ctx context.Context, name string) {
+		if orig.Delete != nil {
+			orig.Delete(ctx, name)
+		}
+		healthStatus.DeleteLabelValues(name)
+	}
+	return cfg
 }