SD-11577: add support for bucketing metrics based on args

Author: rtalvarez

README.md

@@ -35,7 +35,6 @@ Required authentication scopes:
 - `Zone/Firewall Services:Read` is required to fetch zone rule name for `cloudflare_zone_firewall_events_count` metric
 - `Account/Account Rulesets:Read` is required to fetch account rule name for `cloudflare_zone_firewall_events_count` metric
 - `Account:Load Balancing: Monitors and Pools:Read` is required to fetch pools origin health status `cloudflare_pool_origin_health_status` metric
-- `Account/Workers KV Storage:Read` is required for KV metrics
 - `Cloudflare Tunnel Read` is required to fetch Cloudflare Tunnel (Cloudflare Zero Trust) metrics
 
 To authenticate this way, only set `CF_API_TOKEN` (omit `CF_API_EMAIL` and `CF_API_KEY`)
@@ -67,7 +66,7 @@ The exporter can be configured using env variables or command flags.
 | `SCRAPE_DELAY` | scrape delay in seconds, default `300` |
 | `SCRAPE_INTERVAL` | scrape interval in seconds (will query cloudflare every SCRAPE_INTERVAL seconds), default `60` |
 | `METRICS_DENYLIST` | (Optional) cloudflare-exporter metrics to not export, comma delimited list of cloudflare-exporter metrics. If not set, all metrics are exported |
-| `KV_CACHE_INTERVAL` | (Optional) KV namespace cache refresh interval, default `5m` |
+| `KV_NAMESPACE_IDS` | (Optional) KV namespace IDs to track individually, comma delimited. Unlisted namespaces are aggregated as `other` |
 | `ENABLE_PPROF` | (Optional) enable pprof profiling endpoints at `/debug/pprof/`. Accepts `true` or `false`, default `false`. **Warning**: Only enable in development/debugging environments |
 | `ZONE_<NAME>` |  `DEPRECATED since 0.0.5` (optional) Zone ID. Add zones you want to scrape by adding env vars in this format. You can find the zone ids in Cloudflare dashboards. |
 | `LOG_LEVEL` | Set loglevel. Options are error, warn, info, debug. default `error` |
@@ -86,7 +85,7 @@ Corresponding flags:
   -metrics_path="/metrics": path for metrics, default /metrics
   -scrape_delay=300: scrape delay in seconds, defaults to 300
   -scrape_interval=60: scrape interval in seconds, defaults to 60
-  -kv_cache_interval="5m": KV namespace cache refresh interval, default 5 minutes
+  -kv_namespace_ids="": KV namespace IDs to track individually, comma delimited
   -metrics_denylist="": cloudflare-exporter metrics to not export, comma delimited list
   -enable_pprof=false: enable pprof profiling endpoints at /debug/pprof/
   -log_level="error": log level(error,warn,info,debug)

cloudflare.go

@@ -7,7 +7,6 @@ import (
 
 	cf "github.com/cloudflare/cloudflare-go/v4"
 	cfaccounts "github.com/cloudflare/cloudflare-go/v4/accounts"
-	cfkv "github.com/cloudflare/cloudflare-go/v4/kv"
 	cfload_balancers "github.com/cloudflare/cloudflare-go/v4/load_balancers"
 	cfpagination "github.com/cloudflare/cloudflare-go/v4/packages/pagination"
 	cfrulesets "github.com/cloudflare/cloudflare-go/v4/rulesets"
@@ -1110,35 +1109,6 @@ func filterNonFreePlanZones(zones []cfzones.Zone) (filteredZones []cfzones.Zone)
 	return
 }
 
-func fetchKVNamespaces(accountID string) (map[string]string, error) {
-	namespaceMap := make(map[string]string)
-	ctx, cancel := context.WithTimeout(context.Background(), cftimeout)
-	defer cancel()
-	page := cfclient.KV.Namespaces.ListAutoPaging(ctx, cfkv.NamespaceListParams{
-		AccountID: cf.F(accountID),
-	})
-	if page.Err() != nil {
-		return nil, page.Err()
-	}
-
-	seenIDs := make(map[string]struct{})
-	for page.Next() {
-		if page.Err() != nil {
-			log.Errorf("error during paging KV namespaces: %v", page.Err())
-			break
-		}
-		ns := page.Current()
-		if _, exists := seenIDs[ns.ID]; exists {
-			log.Errorf("fetchKVNamespaces: duplicate namespace ID detected (%s), breaking loop", ns.ID)
-			break
-		}
-		seenIDs[ns.ID] = struct{}{}
-		namespaceMap[ns.ID] = ns.Title
-	}
-
-	return namespaceMap, nil
-}
-
 func fetchKVOperations(accountID string) (*cloudflareResponseKV, error) {
 	request := graphql.NewRequest(`
 	query ($accountID: String!, $mintime: Time!, $maxtime: Time!, $limit: Int!) {

main.go

@@ -7,7 +7,6 @@ import (
 	"runtime"
 	"strings"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/nelkinda/health-go"
@@ -27,9 +26,9 @@ var (
 	gql       *GraphQL
 	log       = logrus.New()
 
-	// kvNamespaceCache stores accountID -> namespaceID -> name.
-	// Atomically replaced by refreshKVNamespaceCache, read by getKVNamespaceMap.
-	kvNamespaceCache atomic.Pointer[map[string]map[string]string]
+	// kvTrackedNamespaces is the set of KV namespace IDs that get their own
+	// namespace_id label. All other namespaces are aggregated under "other".
+	kvTrackedNamespaces map[string]struct{}
 )
 
 // var (
@@ -156,30 +155,6 @@ func fetchMetrics(deniedMetricsSet MetricsSet) {
 	wg.Wait()
 }
 
-func refreshKVNamespaceCache() {
-	accounts := fetchAccounts()
-	newCache := make(map[string]map[string]string, len(accounts))
-	for _, a := range accounts {
-		nsMap, err := fetchKVNamespaces(a.ID)
-		if err != nil {
-			log.Warnf("failed to refresh KV namespace cache for account %s: %v", a.ID, err)
-			continue
-		}
-		newCache[a.ID] = nsMap
-	}
-
-	kvNamespaceCache.Store(&newCache)
-	log.Info("KV namespace cache refreshed")
-}
-
-func getKVNamespaceMap(accountID string) map[string]string {
-	cache := kvNamespaceCache.Load()
-	if cache == nil {
-		return nil
-	}
-	return (*cache)[accountID]
-}
-
 func runExporter() {
 	cfgMetricsPath := viper.GetString("metrics_path")
 
@@ -203,15 +178,17 @@ func runExporter() {
 	log.Debugf("Metrics set: %v", metricsSet)
 	mustRegisterMetrics(metricsSet)
 
-	// Populate KV namespace cache at boot, then refresh on interval.
-	refreshKVNamespaceCache()
-	kvCacheInterval := viper.GetDuration("kv_cache_interval")
-	log.Info("KV namespace cache refresh interval set to ", kvCacheInterval)
-	go func() {
-		for range time.NewTicker(kvCacheInterval).C {
-			refreshKVNamespaceCache()
+	// Build tracked KV namespace set from config.
+	kvTrackedNamespaces = make(map[string]struct{})
+	if ids := viper.GetString("kv_namespace_ids"); ids != "" {
+		for _, id := range strings.Split(ids, ",") {
+			id = strings.TrimSpace(id)
+			if id != "" {
+				kvTrackedNamespaces[id] = struct{}{}
+			}
 		}
-	}()
+	}
+	log.Infof("Tracking %d KV namespace IDs", len(kvTrackedNamespaces))
 
 	scrapeInterval := time.Duration(viper.GetInt("scrape_interval")) * time.Second
 	log.Info("Scrape interval set to ", scrapeInterval)
@@ -296,9 +273,9 @@ func main() {
 	viper.BindEnv("cf_timeout")
 	viper.SetDefault("cf_timeout", 10*time.Second)
 
-	flags.Duration("kv_cache_interval", 5*time.Minute, "KV namespace cache refresh interval, default 5 minutes")
-	viper.BindEnv("kv_cache_interval")
-	viper.SetDefault("kv_cache_interval", 5*time.Minute)
+	flags.String("kv_namespace_ids", "", "KV namespace IDs to track individually, comma delimited. Unlisted namespaces are aggregated as 'other'")
+	viper.BindEnv("kv_namespace_ids")
+	viper.SetDefault("kv_namespace_ids", "")
 
 	flags.String("metrics_denylist", "", "metrics to not expose, comma delimited list")
 	viper.BindEnv("metrics_denylist")

prometheus.go

@@ -341,12 +341,12 @@ var (
 	kvRequests = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Name: kvRequestsMetricName.String(),
 		Help: "Number of KV operations by namespace and action type",
-	}, []string{"namespace_name", "action_type", "account"})
+	}, []string{"namespace_id", "action_type", "account"})
 
 	kvLatency = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Name: kvLatencyMetricName.String(),
 		Help: "KV operation latency quantiles (milliseconds)",
-	}, []string{"namespace_name", "action_type", "account", "quantile"})
+	}, []string{"namespace_id", "action_type", "account", "quantile"})
 
 	dnsFirewallQueryCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Name: dnsFirewallQueryCountMetricName.String(),
@@ -631,8 +631,6 @@ func fetchKVAnalytics(account cfaccounts.Account, wg *sync.WaitGroup, deniedMetr
 	wg.Add(1)
 	defer wg.Done()
 
-	namespaceMap := getKVNamespaceMap(account.ID)
-
 	r, err := fetchKVOperations(account.ID)
 	if err != nil {
 		log.Error("failed to fetch KV operations for account ", account.ID, ": ", err)
@@ -643,19 +641,19 @@ func fetchKVAnalytics(account cfaccounts.Account, wg *sync.WaitGroup, deniedMetr
 
 	for _, a := range r.Viewer.Accounts {
 		for _, kv := range a.KvOperationsAdaptiveGroups {
-			namespaceName := namespaceMap[kv.Dimensions.NamespaceID]
-			if namespaceName == "" {
-				namespaceName = kv.Dimensions.NamespaceID
+			nsID := kv.Dimensions.NamespaceID
+			if _, tracked := kvTrackedNamespaces[nsID]; !tracked {
+				nsID = "other"
 			}
 
 			if !deniedMetricsSet.Has(kvRequestsMetricName) {
-				kvRequests.With(prometheus.Labels{"namespace_name": namespaceName, "action_type": kv.Dimensions.ActionType, "account": accountName}).Set(float64(kv.Sum.Requests))
+				kvRequests.With(prometheus.Labels{"namespace_id": nsID, "action_type": kv.Dimensions.ActionType, "account": accountName}).Add(float64(kv.Sum.Requests))
 			}
 			if !deniedMetricsSet.Has(kvLatencyMetricName) {
-				kvLatency.With(prometheus.Labels{"namespace_name": namespaceName, "action_type": kv.Dimensions.ActionType, "account": accountName, "quantile": "P50"}).Set(float64(kv.Quantiles.LatencyMsP50))
-				kvLatency.With(prometheus.Labels{"namespace_name": namespaceName, "action_type": kv.Dimensions.ActionType, "account": accountName, "quantile": "P75"}).Set(float64(kv.Quantiles.LatencyMsP75))
-				kvLatency.With(prometheus.Labels{"namespace_name": namespaceName, "action_type": kv.Dimensions.ActionType, "account": accountName, "quantile": "P99"}).Set(float64(kv.Quantiles.LatencyMsP99))
-				kvLatency.With(prometheus.Labels{"namespace_name": namespaceName, "action_type": kv.Dimensions.ActionType, "account": accountName, "quantile": "P999"}).Set(float64(kv.Quantiles.LatencyMsP999))
+				kvLatency.With(prometheus.Labels{"namespace_id": nsID, "action_type": kv.Dimensions.ActionType, "account": accountName, "quantile": "P50"}).Set(float64(kv.Quantiles.LatencyMsP50))
+				kvLatency.With(prometheus.Labels{"namespace_id": nsID, "action_type": kv.Dimensions.ActionType, "account": accountName, "quantile": "P75"}).Set(float64(kv.Quantiles.LatencyMsP75))
+				kvLatency.With(prometheus.Labels{"namespace_id": nsID, "action_type": kv.Dimensions.ActionType, "account": accountName, "quantile": "P99"}).Set(float64(kv.Quantiles.LatencyMsP99))
+				kvLatency.With(prometheus.Labels{"namespace_id": nsID, "action_type": kv.Dimensions.ActionType, "account": accountName, "quantile": "P999"}).Set(float64(kv.Quantiles.LatencyMsP999))
 			}
 		}
 	}