Merge pull request #2858 from Kavinjsir/feat/cutom-metrics-experiment

✨ Experimental practice to scaffold dashboard manifest for custom metrics

Author: camilamacedo86

docs/book/src/plugins/grafana-v1-alpha.md

@@ -132,6 +132,48 @@ See an example of how to use the plugin in your project:
   - Per-second rate of retries handled by workqueue
 - Sample: <img width="914" src="https://user-images.githubusercontent.com/18136486/180360101-411c81e9-d54e-4b21-bbb0-e3f94fcf48cb.png">
 
+### Visualize Custom Metrics
+
+The Grafana plugin supports scaffolding manifests for custom metrics.
+
+#### Generate Config Template
+
+When the plugin is triggered for the first time, `grafana/custom-metrics/config.yaml` is generated.
+
+```yaml
+---
+customMetrics:
+#  - metric: # Raw custom metric (required)
+#    type:   # Metric type: counter/gauge/histogram (required)
+#    expr:   # Prom_ql for the metric (optional)
+```
+
+#### Add Custom Metrics to Config
+
+You can enter multiple custom metrics in the file. For each element, you need to specify the `metric` and its `type`.
+The Grafana plugin can automatically generate `expr` for visualization.
+Alternatively, you can provide `expr` and the plugin will use the specified one directly.
+
+```yaml
+---
+customMetrics:
+  - metric: memcached_operator_reconcile_total # Raw custom metric (required)
+    type: counter # Metric type: counter/gauge/histogram (required)
+  - metric: memcached_operator_reconcile_time_seconds_bucket
+    type: histogram
+```
+
+#### Scaffold Manifest
+
+Once `config.yaml` is configured, you can run `kubebuilder edit --plugins grafana.kubebuilder.io/v1-alpha` again.
+This time, the plugin will generate `grafana/custom-metrics/custom-metrics-dashboard.json`, which can be imported to Grafana UI.
+
+#### Show case:
+
+See an example of how to visualize your custom metrics:
+
+![output2](https://user-images.githubusercontent.com/18136486/186933170-d2e0de71-e079-4d1b-906a-99a549d66ebf.gif)
+
 ## Subcommands
 
 The Grafana plugin implements the following subcommands:

pkg/plugin/util/util.go

@@ -242,3 +242,14 @@ func ReplaceRegexInFile(path, match, replace string) error {
 	}
 	return nil
 }
+
+// HasFileContentWith check if given `text` can be found in file
+func HasFileContentWith(path, text string) (bool, error) {
+	// nolint:gosec
+	contents, err := ioutil.ReadFile(path)
+	if err != nil {
+		return false, err
+	}
+
+	return strings.Contains(string(contents), text), nil
+}

pkg/plugins/optional/grafana/v1alpha/scaffolds/edit.go

@@ -18,14 +18,22 @@ package scaffolds
 
 import (
 	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"strings"
 
 	"sigs.k8s.io/kubebuilder/v3/pkg/machinery"
 	"sigs.k8s.io/kubebuilder/v3/pkg/plugins"
 	"sigs.k8s.io/kubebuilder/v3/pkg/plugins/optional/grafana/v1alpha/scaffolds/internal/templates"
+
+	"sigs.k8s.io/yaml"
 )
 
 var _ plugins.Scaffolder = &editScaffolder{}
 
+const configFilePath = "grafana/custom-metrics/config.yaml"
+
 type editScaffolder struct {
 	// fs is the filesystem that will be used by the scaffolder
 	fs machinery.Filesystem
@@ -41,15 +49,121 @@ func (s *editScaffolder) InjectFS(fs machinery.Filesystem) {
 	s.fs = fs
 }
 
+func fileExist(configFilePath string) bool {
+	if _, err := os.Stat(configFilePath); os.IsNotExist(err) {
+		return false
+	}
+	return true
+}
+
+func loadConfig(configPath string) ([]templates.CustomMetricItem, error) {
+	if !fileExist(configPath) {
+		return nil, nil
+	}
+
+	// nolint:gosec
+	f, err := os.Open(configPath)
+	if err != nil {
+		return nil, fmt.Errorf("error loading plugin config: %w", err)
+	}
+
+	items, err := configReader(f)
+
+	if err := f.Close(); err != nil {
+		return nil, fmt.Errorf("could not close config.yaml: %w", err)
+	}
+
+	return items, err
+}
+
+func configReader(reader io.Reader) ([]templates.CustomMetricItem, error) {
+	yamlFile, err := ioutil.ReadAll(reader)
+	if err != nil {
+		return nil, err
+	}
+
+	config := templates.CustomMetricsConfig{}
+
+	err = yaml.Unmarshal(yamlFile, &config)
+	if err != nil {
+		return nil, err
+	}
+
+	validatedMetricItems := validateCustomMetricItems(config.CustomMetrics)
+
+	return validatedMetricItems, nil
+}
+
+func validateCustomMetricItems(rawItems []templates.CustomMetricItem) []templates.CustomMetricItem {
+	// 1. Filter items of missing `Metric` or `Type`
+	filterResult := []templates.CustomMetricItem{}
+	for _, item := range rawItems {
+		if hasFields(item) {
+			filterResult = append(filterResult, item)
+		}
+	}
+
+	// 2. Fill Expr if missing
+	validatedItems := make([]templates.CustomMetricItem, len(filterResult))
+	for i, item := range filterResult {
+		validatedItems[i] = fillMissingExpr(item)
+	}
+
+	return validatedItems
+}
+
+func hasFields(item templates.CustomMetricItem) bool {
+	// If `Expr` exists, return true
+	if item.Expr != "" {
+		return true
+	}
+
+	// If `Metric` & valid `Type` exists, return true
+	metricType := strings.ToLower(item.Type)
+	if item.Metric != "" && (metricType == "counter" || metricType == "gauge" || metricType == "histogram") {
+		return true
+	}
+
+	return false
+}
+
+// TODO: Prom_ql exprs can improved to be more pratical and applicable
+func fillMissingExpr(item templates.CustomMetricItem) templates.CustomMetricItem {
+	if item.Expr == "" {
+		switch strings.ToLower(item.Type) {
+		case "counter":
+			item.Expr = "sum(rate(" + item.Metric + `{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, pod)`
+		case "histogram":
+			// nolint: lll
+			item.Expr = "histogram_quantile(0.90, sum by(instance, le) (rate(" + item.Metric + `{job=\"$job\", namespace=\"$namespace\"}[5m])))`
+		default: // gauge
+			item.Expr = item.Metric
+		}
+	}
+	return item
+}
+
 // Scaffold implements cmdutil.Scaffolder
 func (s *editScaffolder) Scaffold() error {
 	fmt.Println("Generating Grafana manifests to visualize controller status...")
 
 	// Initialize the machinery.Scaffold that will write the files to disk
 	scaffold := machinery.NewScaffold(s.fs)
 
-	return scaffold.Execute(
+	configPath := string(configFilePath)
+
+	var templatesBuilder = []machinery.Builder{
 		&templates.RuntimeManifest{},
 		&templates.ResourcesManifest{},
-	)
+		&templates.CustomMetricsConfigManifest{ConfigPath: configPath},
+	}
+
+	configItems, err := loadConfig(configPath)
+	if err == nil && len(configItems) > 0 {
+		templatesBuilder = append(templatesBuilder, &templates.CustomMetricsDashManifest{Items: configItems})
+	} else if err != nil {
+		fmt.Fprintf(os.Stderr, "Error on scaffolding manifest for custom metris:\n%v", err)
+	}
+
+	return scaffold.Execute(templatesBuilder...)
 }

pkg/plugins/optional/grafana/v1alpha/scaffolds/init.go

@@ -51,5 +51,6 @@ func (s *initScaffolder) Scaffold() error {
 	return scaffold.Execute(
 		&templates.RuntimeManifest{},
 		&templates.ResourcesManifest{},
+		&templates.CustomMetricsConfigManifest{ConfigPath: string(configFilePath)},
 	)
 }

pkg/plugins/optional/grafana/v1alpha/scaffolds/internal/templates/custom.go

@@ -0,0 +1,56 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package templates
+
+import (
+	"sigs.k8s.io/kubebuilder/v3/pkg/machinery"
+)
+
+var _ machinery.Template = &CustomMetricsConfigManifest{}
+
+// Kustomization scaffolds a file that defines the kustomization scheme for the prometheus folder
+type CustomMetricsConfigManifest struct {
+	machinery.TemplateMixin
+	ConfigPath string
+}
+
+// SetTemplateDefaults implements file.Template
+func (f *CustomMetricsConfigManifest) SetTemplateDefaults() error {
+	f.Path = f.ConfigPath
+
+	f.TemplateBody = customMetricsConfigTemplate
+
+	f.IfExistsAction = machinery.SkipFile
+
+	return nil
+}
+
+// nolint: lll
+const customMetricsConfigTemplate = `---
+customMetrics:
+#  - metric: # Raw custom metric (required)
+#    type:   # Metric type: counter/gauge/histogram (required)
+#    expr:   # Prom_ql for the metric (optional)
+#
+#
+# Example:
+# ---
+# customMetrics:
+#   - metric: foo_bar
+#     type: histogram
+#   	expr: histogram_quantile(0.90, sum by(instance, le) (rate(foo_bar{job=\"$job\", namespace=\"$namespace\"}[5m])))
+`