SD-11577: decouple kv namespace from scrape interval

Author: rtalvarez

README.md

@@ -67,6 +67,7 @@ The exporter can be configured using env variables or command flags.
 | `SCRAPE_DELAY` | scrape delay in seconds, default `300` |
 | `SCRAPE_INTERVAL` | scrape interval in seconds (will query cloudflare every SCRAPE_INTERVAL seconds), default `60` |
 | `METRICS_DENYLIST` | (Optional) cloudflare-exporter metrics to not export, comma delimited list of cloudflare-exporter metrics. If not set, all metrics are exported |
+| `KV_CACHE_INTERVAL` | (Optional) KV namespace cache refresh interval, default `5m` |
 | `ENABLE_PPROF` | (Optional) enable pprof profiling endpoints at `/debug/pprof/`. Accepts `true` or `false`, default `false`. **Warning**: Only enable in development/debugging environments |
 | `ZONE_<NAME>` |  `DEPRECATED since 0.0.5` (optional) Zone ID. Add zones you want to scrape by adding env vars in this format. You can find the zone ids in Cloudflare dashboards. |
 | `LOG_LEVEL` | Set loglevel. Options are error, warn, info, debug. default `error` |
@@ -85,6 +86,7 @@ Corresponding flags:
   -metrics_path="/metrics": path for metrics, default /metrics
   -scrape_delay=300: scrape delay in seconds, defaults to 300
   -scrape_interval=60: scrape interval in seconds, defaults to 60
+  -kv_cache_interval="5m": KV namespace cache refresh interval, default 5 minutes
   -metrics_denylist="": cloudflare-exporter metrics to not export, comma delimited list
   -enable_pprof=false: enable pprof profiling endpoints at /debug/pprof/
   -log_level="error": log level(error,warn,info,debug)

main.go

@@ -7,6 +7,7 @@ import (
 	"runtime"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/nelkinda/health-go"
@@ -25,6 +26,10 @@ var (
 	cftimeout time.Duration
 	gql       *GraphQL
 	log       = logrus.New()
+
+	// kvNamespaceCache stores accountID -> namespaceID -> name.
+	// Atomically replaced by refreshKVNamespaceCache, read by getKVNamespaceMap.
+	kvNamespaceCache atomic.Pointer[map[string]map[string]string]
 )
 
 // var (
@@ -151,6 +156,30 @@ func fetchMetrics(deniedMetricsSet MetricsSet) {
 	wg.Wait()
 }
 
+func refreshKVNamespaceCache() {
+	accounts := fetchAccounts()
+	newCache := make(map[string]map[string]string, len(accounts))
+	for _, a := range accounts {
+		nsMap, err := fetchKVNamespaces(a.ID)
+		if err != nil {
+			log.Warnf("failed to refresh KV namespace cache for account %s: %v", a.ID, err)
+			continue
+		}
+		newCache[a.ID] = nsMap
+	}
+
+	kvNamespaceCache.Store(&newCache)
+	log.Info("KV namespace cache refreshed")
+}
+
+func getKVNamespaceMap(accountID string) map[string]string {
+	cache := kvNamespaceCache.Load()
+	if cache == nil {
+		return nil
+	}
+	return (*cache)[accountID]
+}
+
 func runExporter() {
 	cfgMetricsPath := viper.GetString("metrics_path")
 
@@ -174,6 +203,16 @@ func runExporter() {
 	log.Debugf("Metrics set: %v", metricsSet)
 	mustRegisterMetrics(metricsSet)
 
+	// Populate KV namespace cache at boot, then refresh on interval.
+	refreshKVNamespaceCache()
+	kvCacheInterval := viper.GetDuration("kv_cache_interval")
+	log.Info("KV namespace cache refresh interval set to ", kvCacheInterval)
+	go func() {
+		for range time.NewTicker(kvCacheInterval).C {
+			refreshKVNamespaceCache()
+		}
+	}()
+
 	scrapeInterval := time.Duration(viper.GetInt("scrape_interval")) * time.Second
 	log.Info("Scrape interval set to ", scrapeInterval)
 
@@ -257,6 +296,10 @@ func main() {
 	viper.BindEnv("cf_timeout")
 	viper.SetDefault("cf_timeout", 10*time.Second)
 
+	flags.Duration("kv_cache_interval", 5*time.Minute, "KV namespace cache refresh interval, default 5 minutes")
+	viper.BindEnv("kv_cache_interval")
+	viper.SetDefault("kv_cache_interval", 5*time.Minute)
+
 	flags.String("metrics_denylist", "", "metrics to not expose, comma delimited list")
 	viper.BindEnv("metrics_denylist")
 	viper.SetDefault("metrics_denylist", "")

prometheus.go

@@ -631,11 +631,7 @@ func fetchKVAnalytics(account cfaccounts.Account, wg *sync.WaitGroup, deniedMetr
 	wg.Add(1)
 	defer wg.Done()
 
-	namespaceMap, err := fetchKVNamespaces(account.ID)
-	if err != nil {
-		log.Error("failed to fetch KV namespaces for account ", account.ID, ": ", err)
-		return
-	}
+	namespaceMap := getKVNamespaceMap(account.ID)
 
 	r, err := fetchKVOperations(account.ID)
 	if err != nil {