Expose lag as prometheus metric (#379)

Author: benbjohnson

cmd/litefs/mount_linux.go

@@ -359,6 +359,10 @@ func (c *MountCommand) initStore(ctx context.Context) error {
 	if err := c.initStoreBackupClient(ctx); err != nil {
 		return err
 	}
+
+	// Initialize as a singleton so we can automatically collect metrics.
+	litefs.GlobalStore.Store(c.Store)
+
 	return nil
 }
 

db.go

@@ -3602,7 +3602,7 @@ var (
 	}, []string{"db"})
 
 	dbLatencySecondsMetricVec = promauto.NewGaugeVec(prometheus.GaugeOpts{
-		Name: "litefs_db_latency_seconds",
+		Name: "litefs_db_lag_seconds",
 		Help: "Latency between generating an LTX file and consuming it.",
 	}, []string{"db"})
 )

store.go

@@ -45,8 +45,15 @@ const (
 	DefaultBackupFullSyncInterval = 10 * time.Second
 )
 
+const (
+	MetricsMonitorInterval = 1 * time.Second
+)
+
 var ErrStoreClosed = fmt.Errorf("store closed")
 
+// GlobalStore represents a single store used for metrics collection.
+var GlobalStore atomic.Value
+
 // Store represents a collection of databases.
 type Store struct {
 	mu   sync.Mutex
@@ -1595,6 +1602,18 @@ func (s *Store) setPrimaryTimestamp(ts int64) {
 	}
 }
 
+// Lag returns the number of seconds that the local instance is lagging
+// behind the primary node. Returns 0 if the node is the primary or if the
+// node is not marked as ready yet.
+func (s *Store) Lag() time.Duration {
+	switch ts := s.PrimaryTimestamp(); ts {
+	case 0, -1:
+		return 0 // primary or not ready
+	default:
+		return time.Duration(time.Now().UnixMilli()-ts) * time.Millisecond
+	}
+}
+
 // Expvar returns a variable for debugging output.
 func (s *Store) Expvar() expvar.Var { return (*StoreVar)(s) }
 
@@ -1821,4 +1840,14 @@ var (
 		Name: "litefs_subscriber_count",
 		Help: "Number of connected subscribers",
 	})
+
+	_ = promauto.NewGaugeFunc(prometheus.GaugeOpts{
+		Name: "litefs_lag_seconds",
+		Help: "Lag behind the primary node, in seconds",
+	}, func() float64 {
+		if s := GlobalStore.Load(); s != nil {
+			return s.(*Store).Lag().Seconds()
+		}
+		return 0
+	})
 )