Fix: Ensure metrics are properly exported with OpenTelemetry

The issue was that metrics were being created during package init() before
the meter provider was set up with the Prometheus exporter.

Changes:
1. Modified provider.go to use lazy initialization and allow SetupMeterProvider()
   to be called explicitly after all readers are added
2. Changed problemmetrics to use lazy initialization via an interface, deferring
   metric creation until first use
3. Reordered main initialization: exporters are now set up first, then
   SetupMeterProvider() is called, then problem daemons are initialized

This ensures that when metrics are created, the meter provider already has
all the configured readers/exporters attached.

Co-authored-by: MartinForReal <[email protected]>

Author: Copilot

cmd/nodeproblemdetector/node_problem_detector.go

@@ -30,6 +30,7 @@ import (
 	"k8s.io/node-problem-detector/pkg/problemdaemon"
 	"k8s.io/node-problem-detector/pkg/problemdetector"
 	"k8s.io/node-problem-detector/pkg/types"
+	"k8s.io/node-problem-detector/pkg/util/metrics"
 	"k8s.io/node-problem-detector/pkg/version"
 )
 
@@ -43,13 +44,8 @@ func npdMain(ctx context.Context, npdo *options.NodeProblemDetectorOptions) erro
 	npdo.SetConfigFromDeprecatedOptionsOrDie()
 	npdo.ValidOrDie()
 
-	// Initialize problem daemons.
-	problemDaemons := problemdaemon.NewProblemDaemons(npdo.MonitorConfigPaths)
-	if len(problemDaemons) == 0 {
-		klog.Fatalf("No problem daemon is configured")
-	}
-
-	// Initialize exporters.
+	// Initialize exporters first so that metric readers are registered before
+	// the meter provider is set up.
 	defaultExporters := []types.Exporter{}
 	if ke := k8sexporter.NewExporterOrDie(ctx, npdo); ke != nil {
 		defaultExporters = append(defaultExporters, ke)
@@ -70,6 +66,16 @@ func npdMain(ctx context.Context, npdo *options.NodeProblemDetectorOptions) erro
 		klog.Fatalf("No exporter is successfully setup")
 	}
 
+	// Setup the meter provider after all exporters have registered their readers.
+	// This ensures metrics are properly exported to all configured backends.
+	metrics.SetupMeterProvider()
+
+	// Initialize problem daemons after meter provider is set up.
+	problemDaemons := problemdaemon.NewProblemDaemons(npdo.MonitorConfigPaths)
+	if len(problemDaemons) == 0 {
+		klog.Fatalf("No problem daemon is configured")
+	}
+
 	// Initialize NPD core.
 	p := problemdetector.NewProblemDetector(problemDaemons, npdExporters)
 	return p.Run(ctx)

pkg/problemmetrics/problem_metrics.go

@@ -29,10 +29,34 @@ import (
 // GlobalProblemMetricsManager is a singleton of ProblemMetricsManager,
 // which should be used to manage all problem-converted metrics across all
 // problem daemons.
-var GlobalProblemMetricsManager *ProblemMetricsManager
+var GlobalProblemMetricsManager ProblemMetricsManagerInterface = &lazyProblemMetricsManager{}
 
-func init() {
-	GlobalProblemMetricsManager = NewProblemMetricsManagerOrDie()
+// ProblemMetricsManagerInterface defines the interface for problem metrics management.
+type ProblemMetricsManagerInterface interface {
+	IncrementProblemCounter(reason string, count int64) error
+	SetProblemGauge(problemType string, reason string, value bool) error
+}
+
+// lazyProblemMetricsManager wraps ProblemMetricsManager with lazy initialization.
+type lazyProblemMetricsManager struct {
+	once    sync.Once
+	manager *ProblemMetricsManager
+}
+
+func (l *lazyProblemMetricsManager) init() {
+	l.once.Do(func() {
+		l.manager = NewProblemMetricsManagerOrDie()
+	})
+}
+
+func (l *lazyProblemMetricsManager) IncrementProblemCounter(reason string, count int64) error {
+	l.init()
+	return l.manager.IncrementProblemCounter(reason, count)
+}
+
+func (l *lazyProblemMetricsManager) SetProblemGauge(problemType string, reason string, value bool) error {
+	l.init()
+	return l.manager.SetProblemGauge(problemType, reason, value)
 }
 
 // ProblemMetricsManager manages problem-converted metrics.

pkg/util/metrics/provider.go

@@ -26,13 +26,11 @@ import (
 )
 
 var (
-	meterProvider     *sdkmetric.MeterProvider
-	meterProviderOnce sync.Once
-	meter             metric.Meter
-	meterOnce         sync.Once
-	readers           []sdkmetric.Reader
-	resources         []*resource.Resource
-	readersMutex      sync.Mutex
+	meterProvider *sdkmetric.MeterProvider
+	readers       []sdkmetric.Reader
+	resources     []*resource.Resource
+	readersMutex  sync.Mutex
+	initialized   bool
 )
 
 // AddReader adds a metric reader to be used when setting up the meter provider.
@@ -54,47 +52,50 @@ func AddReaderWithResource(reader sdkmetric.Reader, res *resource.Resource) {
 
 // SetupMeterProvider initializes the global meter provider with all registered readers.
 // This should be called after all readers have been added.
+// Can be called multiple times safely - only the first call takes effect.
 func SetupMeterProvider() {
-	meterProviderOnce.Do(func() {
-		readersMutex.Lock()
-		defer readersMutex.Unlock()
+	readersMutex.Lock()
+	defer readersMutex.Unlock()
 
-		opts := make([]sdkmetric.Option, 0, len(readers)+1)
-		for _, reader := range readers {
-			opts = append(opts, sdkmetric.WithReader(reader))
-		}
+	if initialized {
+		return
+	}
+	initialized = true
 
-		// Merge all resources if any
-		if len(resources) > 0 {
-			merged := resources[0]
-			for i := 1; i < len(resources); i++ {
-				var err error
-				merged, err = resource.Merge(merged, resources[i])
-				if err != nil {
-					// If merge fails, continue with what we have
-					continue
-				}
+	opts := make([]sdkmetric.Option, 0, len(readers)+1)
+	for _, reader := range readers {
+		opts = append(opts, sdkmetric.WithReader(reader))
+	}
+
+	// Merge all resources if any
+	if len(resources) > 0 {
+		merged := resources[0]
+		for i := 1; i < len(resources); i++ {
+			var err error
+			merged, err = resource.Merge(merged, resources[i])
+			if err != nil {
+				// If merge fails, continue with what we have
+				continue
 			}
-			opts = append(opts, sdkmetric.WithResource(merged))
 		}
+		opts = append(opts, sdkmetric.WithResource(merged))
+	}
 
-		meterProvider = sdkmetric.NewMeterProvider(opts...)
-		otel.SetMeterProvider(meterProvider)
-	})
+	meterProvider = sdkmetric.NewMeterProvider(opts...)
+	otel.SetMeterProvider(meterProvider)
 }
 
 // GetMeter returns the global meter for creating metrics.
+// Note: Metrics created before SetupMeterProvider() is called will use a no-op meter
+// and won't be exported. Always call SetupMeterProvider() after adding all readers.
 func GetMeter() metric.Meter {
-	meterOnce.Do(func() {
-		// Ensure meter provider is set up
-		SetupMeterProvider()
-		meter = otel.Meter("k8s.io/node-problem-detector")
-	})
-	return meter
+	return otel.Meter("k8s.io/node-problem-detector")
 }
 
 // ShutdownMeterProvider gracefully shuts down the meter provider.
 func ShutdownMeterProvider() error {
+	readersMutex.Lock()
+	defer readersMutex.Unlock()
 	if meterProvider != nil {
 		return meterProvider.Shutdown(context.Background())
 	}