NETOBSERV-740: Metrics integration - console plugin frontend (#516)

* Add prom client vendor

* NETOBSERV-739: Add prometheus queries

This introduces prometheus as a new datasource, that can be used either
as a replacement or as a complement of Loki. It doesn't require to
change the frontend/backend API interface: on every frontend query,
backend checks if that query is transposable to prometheus, and if so,
runs it on prometheus, else falls back on loki (if it's enabled)

There's some refactoring of the config and topology handlers to make
place for prometheus.

* Manage DnsFlagsResponseCode

* Use explicit metrics defs; better error message

* Use explicit metrics config; handle prom labels values

* return datasources in query result

* add datasource param & fix tests

* remove disabled check

* manage datasources

* hide filters not available

* Detect available filters without new config

To this by probing with the existing filter encoder function

* Do not show invalid groups

Also, merge all "allowXXX" scope props into a single "allowedScopes"
prop

* fix front lint

* Fix tab tooltip not showing

* run prettier

* Fix main remaining issues for prom datasource (backend)

- Manage FlowDirection: first check if query can be performed using a
  metric agnostic to direction (Any); else, check both Ingress and
Egress metric, and combine them in promQL with OR
- Handle "getNamesForPrefix" with prom
- Split "getTopologyFlows" in 2 parts for cyclo cplx
- Improve error messages when query can't be performed using prometheus
- Add tests

* Do not show prom-unsupported as a Loki error

* Fixes & address feedback

- Consider known metrics labels when fetching label values from prom (else use Loki when possible, or raise error)
- Avoid aggregate on app with prom
- Fix issue when direction is unset in metrics defs
- Fix datasources stats not merged on multiple queries
- Add tests

* Adding unit test on groups filtering

* Add prom measurement metric

* NETOBSERV-1652: retry with Loki on prom 401/403

* Fix percentile queries, metric name has no suffix

In a previous implementation, metric names came with their "_bucket"
prefix for histograms; now this isn't the case anymore but the query
generation wasn't updated accordingly

---------

Co-authored-by: Joel Takvorian <[email protected]>

Author: jpinsonneau

cmd/plugin-backend.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"flag"
 	"fmt"
 	"os"
@@ -43,6 +44,10 @@ func main() {
 	if err != nil {
 		log.WithError(err).Fatal("error reading config file")
 	}
+	err = cfg.Validate()
+	if err != nil {
+		log.WithError(err).Fatal("invalid config")
+	}
 
 	checker, err := cfg.GetAuthChecker()
 	if err != nil {
@@ -55,5 +60,5 @@ func main() {
 		KeyPath:  cfg.Server.KeyPath,
 	})
 
-	server.Start(cfg, checker)
+	server.Start(context.Background(), cfg, checker)
 }

config/sample-config.yaml

@@ -1,6 +1,7 @@
 server:
   port: 9001
   metricsPort: 9002
+  authCheck: none
 loki:
   url: http://localhost:3100
   labels:
@@ -18,8 +19,25 @@ loki:
     #    - SrcK8S_Zone
     #    - DstK8S_Zone
   tenantID: netobserv
-  authCheck: none
   useMocks: false
+prometheus:
+  url: http://prometheus:9090
+  timeout: 30s
+  metrics:
+  - enabled: false
+    name: netobserv_node_egress_bytes_total
+    type: counter
+    valueField: Bytes
+    labels:
+    - SrcK8S_HostName
+    - DstK8S_HostName
+  - enabled: true
+    name: netobserv_node_ingress_bytes_total
+    type: counter
+    valueField: Bytes
+    labels:
+    - SrcK8S_HostName
+    - DstK8S_HostName
 frontend:
   recordTypes:
     - flowLog

go.mod

@@ -32,10 +32,12 @@ require (
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
+	github.com/jpillora/backoff v1.0.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.6.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect

go.sum

@@ -39,6 +39,8 @@ github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
 github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
@@ -59,6 +61,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4=
 github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o=
 github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg=

pkg/config/config.go

@@ -1,6 +1,7 @@
 package config
 
 import (
+	"errors"
 	"fmt"
 	"net/url"
 	"os"
@@ -9,6 +10,7 @@ import (
 
 	"github.com/netobserv/network-observability-console-plugin/pkg/kubernetes/auth"
 	"github.com/netobserv/network-observability-console-plugin/pkg/kubernetes/client"
+	"github.com/netobserv/network-observability-console-plugin/pkg/utils/constants"
 	"github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v3"
 )
@@ -26,6 +28,34 @@ type Server struct {
 	CORSMethods string `yaml:"corsMethods,omitempty" json:"corsMethods,omitempty"`
 	CORSHeaders string `yaml:"corsHeaders,omitempty" json:"corsHeaders,omitempty"`
 	CORSMaxAge  string `yaml:"corsMaxAge,omitempty" json:"corsMaxAge,omitempty"`
+	AuthCheck   string `yaml:"authCheck,omitempty" json:"authCheck,omitempty"`
+}
+
+type Prometheus struct {
+	URL              string       `yaml:"url" json:"url"`
+	Timeout          Duration     `yaml:"timeout,omitempty" json:"timeout,omitempty"`
+	TokenPath        string       `yaml:"tokenPath,omitempty" json:"tokenPath,omitempty"`
+	SkipTLS          bool         `yaml:"skipTls,omitempty" json:"skipTls,omitempty"`
+	CAPath           string       `yaml:"caPath,omitempty" json:"caPath,omitempty"`
+	ForwardUserToken bool         `yaml:"forwardUserToken,omitempty" json:"forwardUserToken,omitempty"`
+	Metrics          []MetricInfo `yaml:"metrics,omitempty" json:"metrics,omitempty"`
+}
+
+type FlowDirection string
+
+const (
+	Egress       FlowDirection = "Egress"
+	Ingress      FlowDirection = "Ingress"
+	AnyDirection FlowDirection = "Any"
+)
+
+type MetricInfo struct {
+	Enabled    bool          `yaml:"enabled" json:"enabled"`
+	Name       string        `yaml:"name,omitempty" json:"name,omitempty"`
+	Type       string        `yaml:"type,omitempty" json:"type,omitempty"`
+	ValueField string        `yaml:"valueField,omitempty" json:"valueField,omitempty"`
+	Direction  FlowDirection `yaml:"direction,omitempty" json:"direction,omitempty"`
+	Labels     []string      `yaml:"labels,omitempty" json:"labels,omitempty"`
 }
 
 type PortNaming struct {
@@ -50,10 +80,9 @@ type Column struct {
 }
 
 type Filter struct {
-	ID        string `yaml:"id" json:"id"`
-	Name      string `yaml:"name" json:"name"`
-	Component string `yaml:"component" json:"component"`
-
+	ID                     string `yaml:"id" json:"id"`
+	Name                   string `yaml:"name" json:"name"`
+	Component              string `yaml:"component" json:"component"`
 	Category               string `yaml:"category,omitempty" json:"category,omitempty"`
 	AutoCompleteAddsQuotes bool   `yaml:"autoCompleteAddsQuotes,omitempty" json:"autoCompleteAddsQuotes,omitempty"`
 	Hint                   string `yaml:"hint,omitempty" json:"hint,omitempty"`
@@ -96,13 +125,16 @@ type Frontend struct {
 	Features        []string      `yaml:"features" json:"features"`
 	Deduper         Deduper       `yaml:"deduper" json:"deduper"`
 	Fields          []FieldConfig `yaml:"fields" json:"fields"`
+	DataSources     []string      `yaml:"dataSources" json:"dataSources"`
+	PromLabels      []string      `yaml:"promLabels" json:"promLabels"`
 }
 
 type Config struct {
-	Loki     Loki     `yaml:"loki" json:"loki"`
-	Frontend Frontend `yaml:"frontend" json:"frontend"`
-	Server   Server   `yaml:"server,omitempty" json:"server,omitempty"`
-	Path     string   `yaml:"-" json:"-"`
+	Loki       Loki       `yaml:"loki" json:"loki"`
+	Prometheus Prometheus `yaml:"prometheus" json:"prometheus"`
+	Frontend   Frontend   `yaml:"frontend" json:"frontend"`
+	Server     Server     `yaml:"server,omitempty" json:"server,omitempty"`
+	Path       string     `yaml:"-" json:"-"`
 }
 
 func ReadFile(version, date, filename string) (*Config, error) {
@@ -114,10 +146,13 @@ func ReadFile(version, date, filename string) (*Config, error) {
 			MetricsPort: 9002,
 			CORSOrigin:  "*",
 			CORSHeaders: "Origin, X-Requested-With, Content-Type, Accept",
+			AuthCheck:   "auto",
 		},
 		Loki: Loki{
-			Timeout:   Duration{Duration: 30 * time.Second},
-			AuthCheck: "auto",
+			Timeout: Duration{Duration: 30 * time.Second},
+		},
+		Prometheus: Prometheus{
+			Timeout: Duration{Duration: 30 * time.Second},
 		},
 		Frontend: Frontend{
 			BuildVersion: version,
@@ -141,6 +176,8 @@ func ReadFile(version, date, filename string) (*Config, error) {
 				{Name: "SrcAddr", Type: "string"},
 				{Name: "DstAddr", Type: "string"},
 			},
+			DataSources: []string{},
+			PromLabels:  []string{},
 		},
 	}
 	if len(filename) == 0 {
@@ -155,56 +192,98 @@ func ReadFile(version, date, filename string) (*Config, error) {
 		return nil, err
 	}
 
-	cfg.Validate()
+	if cfg.IsLokiEnabled() {
+		cfg.Frontend.DataSources = append(cfg.Frontend.DataSources, string(constants.DataSourceLoki))
+	}
+
+	if cfg.IsPromEnabled() {
+		cfg.Frontend.DataSources = append(cfg.Frontend.DataSources, string(constants.DataSourceProm))
+		labels := make(map[string]any)
+		for _, m := range cfg.Prometheus.Metrics {
+			if m.Enabled {
+				for _, l := range m.Labels {
+					labels[l] = true
+				}
+			}
+		}
+		for k := range labels {
+			cfg.Frontend.PromLabels = append(cfg.Frontend.PromLabels, k)
+		}
+	}
+
+	return &cfg, err
+}
 
-	return &cfg, nil
+func (c *Config) IsLokiEnabled() bool {
+	return c.Loki.URL != ""
 }
 
-func (c *Config) Validate() {
-	var configErrors []string
+func (c *Config) IsPromEnabled() bool {
+	return c.Prometheus.URL != ""
+}
 
-	// check config required fields
-	if len(c.Loki.Labels) == 0 {
-		configErrors = append(configErrors, "labels cannot be empty")
+func (c *Config) Validate() error {
+	if !c.IsLokiEnabled() && !c.IsPromEnabled() {
+		return errors.New("neither Loki nor Prometheus is configured; at least one of them should have a URL defined")
 	}
 
-	// parse config urls
-	if len(c.Loki.URL) == 0 {
-		configErrors = append(configErrors, "url cannot be empty")
-	} else {
+	var configErrors []string
+
+	if c.IsLokiEnabled() {
+		log.Infof("Loki is enabled (%s)", c.Loki.URL)
+		// check config required fields
+		if len(c.Loki.Labels) == 0 {
+			configErrors = append(configErrors, "labels cannot be empty")
+		}
+
+		// parse config urls
 		_, err := url.Parse(c.Loki.URL)
 		if err != nil {
 			configErrors = append(configErrors, "wrong Loki URL")
 		}
+		if len(c.Loki.StatusURL) > 0 {
+			_, err := url.Parse(c.Loki.StatusURL)
+			if err != nil {
+				configErrors = append(configErrors, "wrong Loki status URL")
+			}
+		}
+	} else {
+		log.Info("Loki is disabled")
 	}
-	if len(c.Loki.StatusURL) > 0 {
-		_, err := url.Parse(c.Loki.StatusURL)
+
+	if c.IsPromEnabled() {
+		log.Infof("Prometheus is enabled (%s)", c.Prometheus.URL)
+		// parse config urls
+		_, err := url.Parse(c.Prometheus.URL)
 		if err != nil {
-			configErrors = append(configErrors, "wrong Loki status URL")
+			configErrors = append(configErrors, "wrong Prometheus URL")
 		}
+	} else {
+		log.Info("Prometheus is disabled")
 	}
 
-	// crash on config errors
 	if len(configErrors) > 0 {
 		configErrors = append([]string{fmt.Sprintf("Config file has %d errors:\n", len(configErrors))}, configErrors...)
-		log.Fatal(strings.Join(configErrors, "\n - "))
+		return errors.New(strings.Join(configErrors, "\n - "))
 	}
+
+	return nil
 }
 
 func (c *Config) GetAuthChecker() (auth.Checker, error) {
 	// parse config auth
 	var checkType auth.CheckType
-	if c.Loki.AuthCheck == "auto" {
-		if c.Loki.ForwardUserToken {
-			// FORWARD lokiAuth mode
-			checkType = auth.CheckAuthenticated
-		} else {
-			// HOST or DISABLED lokiAuth mode
+	if c.Server.AuthCheck == "auto" {
+		// FORWARD mode
+		checkType = auth.CheckAuthenticated
+		if (c.IsLokiEnabled() && !c.Loki.ForwardUserToken) ||
+			(c.IsPromEnabled() && !c.Prometheus.ForwardUserToken) {
+			// HOST or DISABLED mode
 			checkType = auth.CheckAdmin
 		}
 		log.Info(fmt.Sprintf("auth-check 'auto' resolved to '%s'", checkType))
 	} else {
-		checkType = auth.CheckType(c.Loki.AuthCheck)
+		checkType = auth.CheckType(c.Server.AuthCheck)
 	}
 	if checkType == auth.CheckNone {
 		log.Warn("INSECURE: auth checker is disabled")

pkg/config/loki.go

@@ -17,7 +17,6 @@ type Loki struct {
 	StatusUserKeyPath  string   `yaml:"statusUserKeyPath,omitempty" json:"statusUserKeyPath,omitempty"`
 	UseMocks           bool     `yaml:"useMocks,omitempty" json:"useMocks,omitempty"`
 	ForwardUserToken   bool     `yaml:"forwardUserToken,omitempty" json:"forwardUserToken,omitempty"`
-	AuthCheck          string   `yaml:"authCheck,omitempty" json:"authCheck,omitempty"`
 	labelsMap          map[string]struct{}
 }