top metrics (#79)

Author: yuchen-db

cmd/thanos/receive.go

@@ -440,6 +440,30 @@ func runReceive(
 		})
 	}
 
+	level.Debug(logger).Log("msg", "setting up periodic top metrics collection")
+	{
+		topMetricNumSeries := promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{
+			Name: "thanos_receive_top_metric_num_series",
+			Help: "Number of series in top metric.",
+		}, []string{"tenant", "metric_name"})
+		ctx, cancel := context.WithCancel(context.Background())
+		g.Add(func() error {
+			return runutil.Repeat(conf.topMetricsUpdateInterval, ctx.Done(), func() error {
+				level.Error(logger).Log("msg", "getting top metrics")
+				for _, ts := range dbs.TenantStats(conf.numTopMetricsPerTenant, labels.MetricName) {
+					for _, ms := range ts.Stats.IndexPostingStats.CardinalityMetricsStats {
+						if ms.Count >= conf.topMetricsMinimumCardinality {
+							topMetricNumSeries.WithLabelValues(ts.Tenant, ms.Name).Set(float64(ms.Count))
+						}
+					}
+				}
+				return nil
+			})
+		}, func(err error) {
+			cancel()
+		})
+	}
+
 	{
 		if limiter.CanReload() {
 			ctx, cancel := context.WithCancel(context.Background())
@@ -853,6 +877,10 @@ type receiveConfig struct {
 	limitsConfigReloadTimer time.Duration
 
 	asyncForwardWorkerCount uint
+
+	numTopMetricsPerTenant       int
+	topMetricsMinimumCardinality uint64
+	topMetricsUpdateInterval     time.Duration
 }
 
 func (rc *receiveConfig) registerFlag(cmd extkingpin.FlagClause) {
@@ -997,6 +1025,13 @@ func (rc *receiveConfig) registerFlag(cmd extkingpin.FlagClause) {
 	rc.writeLimitsConfig = extflag.RegisterPathOrContent(cmd, "receive.limits-config", "YAML file that contains limit configuration.", extflag.WithEnvSubstitution(), extflag.WithHidden())
 	cmd.Flag("receive.limits-config-reload-timer", "Minimum amount of time to pass for the limit configuration to be reloaded. Helps to avoid excessive reloads.").
 		Default("1s").Hidden().DurationVar(&rc.limitsConfigReloadTimer)
+
+	cmd.Flag("receive.num-top-metrics-per-tenant", "The number of top metrics to track for each tenant.").
+		Default("100").IntVar(&rc.numTopMetricsPerTenant)
+	cmd.Flag("receive.top-metrics-minimum-cardinality", "The minimum cardinality for a metric to be considered top metric.").
+		Default("10000").Uint64Var(&rc.topMetricsMinimumCardinality)
+	cmd.Flag("receive.top-metrics-update-interval", "The interval at which the top metrics are updated.").
+		Default("5m").DurationVar(&rc.topMetricsUpdateInterval)
 }
 
 // determineMode returns the ReceiverMode that this receiver is configured to run in.