[Feature] Global Metrics (#1668)

Author: ajanikow

CHANGELOG.md

@@ -18,6 +18,7 @@
 - (Feature) (Analytics) Metadata
 - (Feature) (Analytics) StatefulSet
 - (Feature) Imported ArangoBackup Cleanup
+- (Feature) Global Metrics
 
 ## [1.2.40](https://github.com/arangodb/kube-arangodb/tree/1.2.40) (2024-04-10)
 - (Feature) Add Core fields to the Scheduler Container Spec

Makefile

@@ -703,6 +703,7 @@ run-unit-tests: $(SOURCES)
 		$(REPOPATH)/pkg/storage/... \
 	    $(REPOPATH)/pkg/crd/... \
 		$(REPOPATH)/pkg/util/... \
+		$(REPOPATH)/pkg/generated/metric_descriptions/... \
 		$(REPOPATH)/cmd/... \
 		$(REPOPATH)/pkg/handlers/...
 

docs/generated/metrics/README.md

@@ -14,6 +14,7 @@ has_toc: false
 
 |                                                                 Name                                                                  |     Namespace     |       Group       |  Type   | Description                                                                           |
 |:-------------------------------------------------------------------------------------------------------------------------------------:|:-----------------:|:-----------------:|:-------:|:--------------------------------------------------------------------------------------|
+|                              [arango_operator_objects_processed](./arango_operator_objects_processed.md)                              |  arango_operator  |      objects      | Counter | Number of the processed objects                                                       |
 |                                [arangodb_operator_agency_errors](./arangodb_operator_agency_errors.md)                                | arangodb_operator |      agency       | Counter | Current count of agency cache fetch errors                                            |
 |                               [arangodb_operator_agency_fetches](./arangodb_operator_agency_fetches.md)                               | arangodb_operator |      agency       | Counter | Current count of agency cache fetches                                                 |
 |                                 [arangodb_operator_agency_index](./arangodb_operator_agency_index.md)                                 | arangodb_operator |      agency       |  Gauge  | Current index of the agency cache                                                     |

docs/generated/metrics/arango_operator_objects_processed.md

@@ -0,0 +1,17 @@
+---
+layout: page
+title: arango_operator_objects_processed
+parent: List of available metrics
+---
+
+# arango_operator_objects_processed (Counter)
+
+## Description
+
+Number of the processed objects
+
+## Labels
+
+|     Label     | Description   |
+|:-------------:|:--------------|
+| operator_name | Operator Name |

internal/metrics.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2024 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -38,6 +38,9 @@ var metricsGoTemplate []byte
 //go:embed metrics.item.go.tmpl
 var metricsItemGoTemplate []byte
 
+//go:embed metrics.item.go_test.tmpl
+var metricsItemGoTestTemplate []byte
+
 //go:embed metrics.item.tmpl
 var metricItemTemplate []byte
 
@@ -98,6 +101,8 @@ type Metric struct {
 	Type             string `json:"type" yaml:"type"`
 	ShortDescription string `json:"shortDescription" yaml:"shortDescription"`
 
+	Global bool `json:"global" yaml:"global"`
+
 	Labels        []Label    `json:"labels" yaml:"labels"`
 	AlertingRules []Alerting `json:"alertingRules" yaml:"alertingRules"`
 }
@@ -277,22 +282,22 @@ func generateLabels(labels []Label) string {
 
 func generateMetricsGO(root string, in MetricsDoc) error {
 	i, err := template.New("metrics").Parse(string(metricsItemGoTemplate))
+	if err != nil {
+		return err
+	}
 
+	t, err := template.New("metrics").Parse(string(metricsItemGoTestTemplate))
 	if err != nil {
 		return err
 	}
+
 	for _, namespace := range in.Namespaces.Keys() {
 		for _, g := range in.Namespaces[namespace].Keys() {
 			for _, metric := range in.Namespaces[namespace][g].Keys() {
 				details := in.Namespaces[namespace][g][metric]
 
 				mname := fmt.Sprintf("%s_%s_%s", namespace, g, metric)
 
-				out, err := os.OpenFile(path.Join(root, fmt.Sprintf("%s.go", mname)), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
-				if err != nil {
-					return err
-				}
-
 				parts := strings.Split(mname, "_")
 				tparts := strings.Split(strings.Title(strings.Join(parts, " ")), " ")
 
@@ -311,6 +316,10 @@ func generateMetricsGO(root string, in MetricsDoc) error {
 				params = append(params, "value float64")
 				keys = append(keys, "value")
 
+				var mapTypes = map[string]string{}
+				var mapKeys []string
+				var mapIKeys = map[string]string{}
+
 				for _, label := range details.Labels {
 					v := strings.Split(strings.ToLower(label.Key), "_")
 					for id := range v {
@@ -323,29 +332,60 @@ func generateMetricsGO(root string, in MetricsDoc) error {
 
 					k := strings.Join(v, "")
 
+					v[0] = strings.Title(v[0])
+
+					kPublic := strings.Join(v, "")
+
 					keys = append(keys, k)
+					mapKeys = append(mapKeys, kPublic)
+
+					mapIKeys[kPublic] = k
 
 					if t := label.Type; t != nil {
 						params = append(params, fmt.Sprintf("%s %s", k, *t))
+						mapTypes[kPublic] = *t
 					} else {
 						params = append(params, fmt.Sprintf("%s string", k))
+						mapTypes[kPublic] = "string"
 					}
 				}
 
-				if err := i.Execute(out, map[string]interface{}{
-					"name":             mname,
-					"fname":            strings.Join(fnameParts, ""),
-					"ename":            strings.Join(tparts, ""),
-					"shortDescription": details.ShortDescription,
-					"labels":           generateLabels(details.Labels),
-					"type":             details.Type,
-					"fparams":          strings.Join(params, ", "),
-					"fkeys":            strings.Join(keys, ", "),
-				}); err != nil {
+				save := func(template *template.Template, path string) error {
+					out, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
+					if err != nil {
+						return err
+					}
+
+					if err := template.Execute(out, map[string]interface{}{
+						"name":             mname,
+						"fname":            strings.Join(fnameParts, ""),
+						"ename":            strings.Join(tparts, ""),
+						"shortDescription": details.ShortDescription,
+						"global":           details.Global,
+						"labels":           generateLabels(details.Labels),
+						"type":             details.Type,
+						"mapTypes":         mapTypes,
+						"mapKeys":          mapKeys,
+						"mapIKeys":         mapIKeys,
+						"args":             strings.Join(params[1:], ", "),
+						"fparams":          strings.Join(params, ", "),
+						"fkeys":            strings.Join(keys, ", "),
+					}); err != nil {
+						return err
+					}
+
+					if err := out.Close(); err != nil {
+						return err
+					}
+
+					return nil
+				}
+
+				if err := save(i, path.Join(root, fmt.Sprintf("%s.go", mname))); err != nil {
 					return err
 				}
 
-				if err := out.Close(); err != nil {
+				if err := save(t, path.Join(root, fmt.Sprintf("%s_test.go", mname))); err != nil {
 					return err
 				}
 			}

internal/metrics.go.tmpl

@@ -28,6 +28,9 @@ import (
 var (
 	descriptions []metrics.Description
 	descriptionsLock sync.Mutex
+
+	collectors []metrics.Collector
+	collectorsLock sync.Mutex
 )
 
 func registerDescription( d ... metrics.Description) {
@@ -41,9 +44,29 @@ func registerDescription( d ... metrics.Description) {
 	descriptions = append(descriptions, d...)
 }
 
+func registerCollector( d ... metrics.Collector) {
+	if len(d) == 0 {
+		return
+	}
+
+	collectorsLock.Lock()
+	defer collectorsLock.Unlock()
+
+	collectors = append(collectors, d...)
+}
+
 func Descriptions (c metrics.PushDescription) {
 	descriptionsLock.Lock()
 	defer descriptionsLock.Unlock()
 
 	c.Push(descriptions...)
+}
+
+func Collectors (c metrics.PushMetric) {
+	collectorsLock.Lock()
+	defer collectorsLock.Unlock()
+
+	for _, collector := range collectors {
+		collector.CollectMetrics(c)
+	}
 }

internal/metrics.item.go.tmpl

@@ -1,3 +1,4 @@
+{{- $root := . -}}
 //
 // DISCLAIMER
 //
@@ -20,20 +21,156 @@
 
 package metric_descriptions
 
-import "github.com/arangodb/kube-arangodb/pkg/util/metrics"
+import (
+    "github.com/arangodb/kube-arangodb/pkg/util/metrics"
+{{- if .global }}
+
+	"sync"
+{{- end }}
+)
 
 var (
     {{ .fname }} = metrics.NewDescription("{{ .name }}", "{{ .shortDescription }}", {{ .labels }}, nil)
 )
 
 func init() {
     registerDescription({{ .fname }})
+{{- if .global }}
+    registerCollector({{ .fname }}Global)
+{{- end }}
 }
 
 func {{ .ename }}() metrics.Description {
     return {{ .fname }}
 }
 
+{{- if .global }}
+
+func {{ .ename }}Get({{ .args }}) float64 {
+    return {{ .fname }}Global.Get({{ .ename }}Item{
+{{- range $i, $field := .mapKeys }}
+        {{ $field }}: {{ index $root.mapIKeys $field }},
+{{- end }}
+    })
+}
+
+func {{ .ename }}Add({{ .fparams }}) {
+    {{ .fname }}Global.Add(value, {{ .ename }}Item{
+{{- range $i, $field := .mapKeys }}
+        {{ $field }}: {{ index $root.mapIKeys $field }},
+{{- end }}
+    })
+}
+{{- if eq .type "Counter" }}
+
+func {{ .ename }}Inc({{ .args }}) {
+    {{ .fname }}Global.Inc({{ .ename }}Item{
+{{- range $i, $field := .mapKeys }}
+        {{ $field }}: {{ index $root.mapIKeys $field }},
+{{- end }}
+    })
+}
+{{- end }}
+
+func Get{{ .ename }}Factory() {{ .ename }}Factory {
+    return {{ .fname }}Global
+}
+
+var {{ .fname }}Global = &{{ .fname }}Factory{
+ items: {{ .fname }}Items{},
+}
+
+type {{ .ename }}Factory interface {
+    Get(object {{ .ename }}Item) float64
+    Add(value float64, object {{ .ename }}Item)
+    Remove(object {{ .ename }}Item)
+    Items() []{{ .ename }}Item
+{{- if eq .type "Counter" }}
+
+    Inc(object {{ .ename }}Item)
+{{- end }}
+}
+
+type {{ .fname }}Factory struct {
+    lock sync.RWMutex
+
+    items {{ .fname }}Items
+}
+
+func (a *{{ .fname }}Factory) Get(object {{ .ename }}Item) float64 {
+    a.lock.Lock()
+    defer a.lock.Unlock()
+
+    v, ok := a.items[object]
+    if !ok {
+        return 0
+    }
+
+    return v
+}
+
+func (a *{{ .fname }}Factory) Add(value float64, object {{ .ename }}Item) {
+    a.lock.Lock()
+    defer a.lock.Unlock()
+
+    v, ok := a.items[object]
+    if !ok {
+        a.items[object] = value
+        return
+    }
+
+    a.items[object] = value + v
+}
+
+func (a *{{ .fname }}Factory) Remove(obj {{ .ename }}Item) {
+    a.lock.Lock()
+    defer a.lock.Unlock()
+
+    delete(a.items, obj)
+}
+
+func (a *{{ .fname }}Factory) Items() []{{ .ename }}Item {
+    a.lock.Lock()
+    defer a.lock.Unlock()
+
+    var r = make([]{{ .ename }}Item, 0, len(a.items))
+
+    for k := range a.items {
+        r = append(r, k)
+    }
+
+    return r
+}
+{{- if eq .type "Counter" }}
+
+func (a *{{ .fname }}Factory) Inc(object {{ .ename }}Item) {
+    a.Add(1, object)
+}
+{{- end }}
+
+func (a *{{ .fname }}Factory) CollectMetrics(in metrics.PushMetric) {
+    a.lock.RLock()
+    defer a.lock.RUnlock()
+
+	for k, v := range a.items {
+	    in.Push({{ .fname }}.{{ .type }}(v{{- range .mapKeys }}, k.{{ . }}{{- end }}))
+	}
+}
+
+func (a *{{ .fname }}Factory) CollectDescriptions(in metrics.PushDescription) {
+	in.Push({{ .fname }})
+}
+
+type {{ .fname }}Items map[{{ .ename }}Item]float64
+
+type {{ .ename }}Item struct {
+{{- range .mapKeys }}
+    {{ . }} {{ index $root.mapTypes . }}
+{{- end }}
+}
+{{- else }}
+
 func {{ .ename }}{{ .type }}({{ .fparams }}) metrics.Metric {
     return {{ .ename }}().{{ .type }}({{ .fkeys }})
-}
+}
+{{- end }}