rulers: Add support to persist tokens in rulers (#5987)

* Feat: Add support to store tokens in rulers

Signed-off-by: Raphael Silva <[email protected]>

* Add changelog

Signed-off-by: Raphael Silva <[email protected]>

* Add new flag to v1 guarantees documentation

Signed-off-by: Raphael Silva <[email protected]>

* Fix lint error in documentation

Signed-off-by: Raphael Silva <[email protected]>

* Make changelog consistent with convention

Signed-off-by: Raphael Silva <[email protected]>

* Fix lint error in documentation

Signed-off-by: Raphael Silva <[email protected]>

---------

Signed-off-by: Raphael Silva <[email protected]>

Author: rapphil

CHANGELOG.md

@@ -1,9 +1,9 @@
 # Changelog
 
 ## master / unreleased
-
 * [CHANGE] Upgrade Dockerfile Node version from 14x to 18x. #5906
 * [CHANGE] Ingester: Remove `-querier.query-store-for-labels-enabled` flag. Querying long-term store for labels is always enabled. #5984
+* [ENHANCEMENT] rulers: Add support to persist tokens in rulers. #5987
 * [ENHANCEMENT] Query Frontend/Querier: Added store gateway postings touched count and touched size in Querier stats and log in Query Frontend. #5892
 * [ENHANCEMENT] Query Frontend/Querier: Returns `warnings` on prometheus query responses. #5916
 * [ENHANCEMENT] Ingester: Allowing to configure `-blocks-storage.tsdb.head-compaction-interval` flag up to 30 min and add a jitter on the first head compaction. #5919 #5928

docs/configuration/config-file-reference.md

@@ -4262,6 +4262,11 @@ ring:
   # CLI flag: -ruler.ring.zone-awareness-enabled
   [zone_awareness_enabled: <boolean> | default = false]
 
+  # EXPERIMENTAL: File path where tokens are stored. If empty, tokens are not
+  # stored at shutdown and restored at startup.
+  # CLI flag: -ruler.ring.tokens-file-path
+  [tokens_file_path: <string> | default = ""]
+
   # Name of network interface to read address from.
   # CLI flag: -ruler.ring.instance-interface-names
   [instance_interface_names: <list of string> | default = [eth0 en0]]

docs/configuration/v1-guarantees.md

@@ -109,3 +109,5 @@ Currently experimental features are:
   - `store-gateway.sharding-ring.final-sleep` (duration) CLI flag
   - `alertmanager-sharding-ring.final-sleep` (duration) CLI flag
 - OTLP Receiver
+- Persistent tokens in the Ruler Ring:
+  - `-ruler.ring.tokens-file-path` (path) CLI flag

pkg/ruler/ruler.go

@@ -372,6 +372,7 @@ func enableSharding(r *Ruler, ringStore kv.Client) error {
 	// chained via "next delegate").
 	delegate := ring.BasicLifecyclerDelegate(r)
 	delegate = ring.NewLeaveOnStoppingDelegate(delegate, r.logger)
+	delegate = ring.NewTokensPersistencyDelegate(r.cfg.Ring.TokensFilePath, ring.JOINING, delegate, r.logger)
 	delegate = ring.NewAutoForgetDelegate(r.cfg.Ring.HeartbeatTimeout*ringAutoForgetUnhealthyPeriods, delegate, r.logger)
 
 	rulerRingName := "ruler"

pkg/ruler/ruler_ring.go

@@ -43,6 +43,7 @@ type RingConfig struct {
 	HeartbeatTimeout     time.Duration `yaml:"heartbeat_timeout"`
 	ReplicationFactor    int           `yaml:"replication_factor"`
 	ZoneAwarenessEnabled bool          `yaml:"zone_awareness_enabled"`
+	TokensFilePath       string        `yaml:"tokens_file_path"`
 
 	// Instance details
 	InstanceID             string   `yaml:"instance_id" doc:"hidden"`
@@ -75,6 +76,7 @@ func (cfg *RingConfig) RegisterFlags(f *flag.FlagSet) {
 	f.DurationVar(&cfg.FinalSleep, "ruler.ring.final-sleep", 0*time.Second, "The sleep seconds when ruler is shutting down. Need to be close to or larger than KV Store information propagation delay")
 	f.IntVar(&cfg.ReplicationFactor, "ruler.ring.replication-factor", 1, "EXPERIMENTAL: The replication factor to use when loading rule groups for API HA.")
 	f.BoolVar(&cfg.ZoneAwarenessEnabled, "ruler.ring.zone-awareness-enabled", false, "EXPERIMENTAL: True to enable zone-awareness and load rule groups across different availability zones for API HA.")
+	f.StringVar(&cfg.TokensFilePath, "ruler.ring.tokens-file-path", "", "EXPERIMENTAL: File path where tokens are stored. If empty, tokens are not stored at shutdown and restored at startup.")
 
 	// Instance flags
 	cfg.InstanceInterfaceNames = []string{"eth0", "en0"}