feature: Debug the configuration on the web ui.

Signed-off-by: 孙林耀 <sunlinyao@vip.qq.com>

Author: MicroOps-cn

.gitignore

@@ -12,4 +12,7 @@
 *.out
 dist/
 .idea/
-data_exporter
+data_exporter
+ui/node_modules
+ui/build
+/ui/yarn-error.log

.yamllint.yml

@@ -5,6 +5,7 @@ yaml-files:
 
 ignore: |
   pkg/tail@v1.0.0
+  ui/node_modules
 
 rules:
   braces: enable

Makefile.common

@@ -124,7 +124,7 @@ endif
 %: common-% ;
 
 .PHONY: common-all
-common-all: precheck style check_license lint yamllint unused build test
+common-all: precheck style check_license lint yamllint unused ui build test
 
 .PHONY: common-style
 common-style:
@@ -139,7 +139,7 @@ common-style:
 .PHONY: common-check_license
 common-check_license:
 	@echo ">> checking license header"
-	@licRes=$$(for file in $$(find . -type f -iname '*.go' ! -path './vendor/*' ! -path './pkg/tail@v1.0.0/*') ; do \
+	@licRes=$$(for file in $$(find . -type f -regextype posix-extended -regex ".*\.(go|tsx|css|js|ts)" ! -path './transport/static/*' ! -path './ui/build/*' ! -path './ui/node_modules/*' ! -path './vendor/*' ! -path './pkg/tail@v1.0.0/*') ; do \
                awk 'NR<=3' $$file | grep -Eq "(Copyright|generated|GENERATED)" || echo $$file; \
        done); \
        if [ -n "$${licRes}" ]; then \
@@ -314,4 +314,9 @@ $(1)_precheck:
 		echo "Execution of '$$(PRECHECK_COMMAND_$(1))' command failed. Is $(1) installed?"; \
 		exit 1; \
 	fi
-endef
+endef
+
+.PHONY: ui
+ui:
+	cd ui/ && yarn build
+	rm -rf transport/static && mv ui/build transport/static

README-en.md

@@ -38,14 +38,33 @@ make common-build && docker build -t data_exporter .
 
 #### debug config file
 
+##### Debugging in cli
+
 ```shell
 ./data_exporter --config.path="data_exporter.yaml" --log.level=debug
 ```
+
 Or use docker:
+
 ```bash
 docker run  -it --rm -v `pwd`:/etc/data_exporter/ --name data_exporter microops/data_exporter debug  
 ```
 
+##### Debugging in web ui
+
+```shell
+./data_exporter --config.path="data_exporter.yaml" --web.ui-enable
+```
+
+Or use docker:
+
+```bash
+docker run  -it --rm -v `pwd`:/etc/data_exporter/ -p 9116:9116 --name data_exporter microops/data_exporter --web.ui-enable  
+```
+
+Then visit `http://localhost:9116/-/ui/static/`
+![](./docs/images/debug-ui.png)
+
 #### running examples
 
 ```shell

README.md

@@ -39,14 +39,33 @@ make common-build && docker build -t data_exporter .
 
 #### 调试配置文件
 
+##### 在CLI中调试
+
 ```shell
 ./data_exporter debug --config.path="data_exporter.yaml"
 ```
+
 或者使用Docker:
+
 ```bash
 docker run  -it --rm -v `pwd`:/etc/data_exporter/ --name data_exporter microops/data_exporter debug  
 ```
 
+##### 在Web UI中调试
+
+```shell
+./data_exporter --config.path="data_exporter.yaml" --web.ui-enable
+```
+
+或者使用Docker:
+
+```bash
+docker run  -it --rm -v `pwd`:/etc/data_exporter/ -p 9116:9116 --name data_exporter microops/data_exporter --web.ui-enable  
+```
+
+然后访问 `http://localhost:9116/-/ui/static/`
+![](./docs/images/debug-ui.png)
+
 #### 启动examples
 
 ```shell

VERSION

@@ -1 +1 @@
-0.4.3
+0.5.0

collector/collect.go

@@ -166,19 +166,38 @@ func (c *CollectConfig) GetMetricByDs(ctx context.Context, logger log.Logger, ds
 	}
 }
 func (c *CollectConfig) GetMetric(logger log.Logger, data []byte, rcs RelabelConfigs, metrics chan<- MetricGenerator) {
+	var err error
 	for _, mc := range c.Metrics {
+		var dps []Datapoint
 		rcs = append(rcs, mc.RelabelConfigs...)
 		metricLogger := log.With(logger, "metric", mc.Name)
 		level.Debug(metricLogger).Log("title", "Raw Data", "data_format", c.DataFormat, "data", string(wrapper.Limit[byte](data, 256, wrapper.PosCenter, []byte(" ... ")...)))
 		switch c.DataFormat.ToLower() {
 		case Regex:
-			mc.GetMetricByRegex(metricLogger, data, rcs, metrics)
+			dps = mc.GetDatapointsByRegex(metricLogger, data)
 		case Json:
-			mc.GetMetricByJson(metricLogger, data, rcs, metrics)
+			dps = mc.GetDatapointsByJson(metricLogger, data)
 		case Xml:
-			mc.GetMetricByXml(metricLogger, data, rcs, metrics)
+			dps = mc.GetDatapointsByXml(metricLogger, data)
 		case Yaml:
-			mc.GetMetricByYaml(metricLogger, data, rcs, metrics)
+			dps = mc.GetDatapointsByYaml(metricLogger, data)
+		}
+
+		for _, dp := range dps {
+			m := MetricGenerator{
+				logger:     logger,
+				MetricType: mc.MetricType,
+				Name:       mc.Name,
+				Labels:     Labels{Label{Name: "name", Value: mc.Name}},
+			}
+			for name, val := range dp {
+				m.Labels.Append(name, val)
+			}
+			m.Labels, err = mc.Relabels(logger, rcs, m.Labels)
+			if err != nil {
+				continue
+			}
+			metrics <- m
 		}
 	}
 }
@@ -273,7 +292,7 @@ func (c *CollectContext) Describe(_ chan<- *prometheus.Desc) {
 }
 
 func (c *CollectContext) Collect(proMetrics chan<- prometheus.Metric) {
-	metrics := make(chan MetricGenerator, 10)
+	mgs := NewMetricGenerators(10, c.logger)
 	go func() {
 		wg := sync.WaitGroup{}
 		for i := range c.Datasource {
@@ -291,20 +310,43 @@ func (c *CollectContext) Collect(proMetrics chan<- prometheus.Metric) {
 				wg.Add(1)
 				go func(idx int) {
 					defer wg.Done()
-					c.GetMetricByDs(c.Context, c.logger, &ds, metrics)
+					c.GetMetricByDs(c.Context, c.logger, &ds, mgs.metrics)
 				}(i)
 			}
 		}
 		wg.Wait()
-		close(metrics)
+		close(mgs.metrics)
 	}()
-	for metric := range metrics {
+	mgs.Collect(proMetrics)
+	c.metrics.Collect(proMetrics)
+}
+
+type MetricGenerators struct {
+	metrics chan MetricGenerator
+	logger  log.Logger
+}
+
+func NewMetricGenerators(size int, logger log.Logger) *MetricGenerators {
+	return &MetricGenerators{
+		metrics: make(chan MetricGenerator, 10),
+		logger:  logger,
+	}
+}
+
+func (*MetricGenerators) Describe(_ chan<- *prometheus.Desc) {
+}
+
+func (mgs *MetricGenerators) Ch() chan MetricGenerator {
+	return mgs.metrics
+}
+func (mgs *MetricGenerators) Collect(proMetrics chan<- prometheus.Metric) {
+	for metric := range mgs.metrics {
 		if metric.Labels.Has(LabelMetricValues) {
-			ms, errs := metric.getMetrics()
+			ms, errs := metric.GetMetrics()
 			for _, err := range errs {
 				if err != nil {
 					collectErrorCount.WithLabelValues("metric", metric.Name).Inc()
-					level.Error(log.With(c.logger, "metric", metric.Name)).Log("log", "failed to get prometheus metric", "err", err)
+					level.Error(log.With(mgs.logger, "metric", metric.Name)).Log("log", "failed to get prometheus metric", "err", err)
 				}
 			}
 			for _, m := range ms {
@@ -316,14 +358,12 @@ func (c *CollectContext) Collect(proMetrics chan<- prometheus.Metric) {
 			m, err := metric.getMetric()
 			if err != nil {
 				collectErrorCount.WithLabelValues("metric", metric.Name).Inc()
-				level.Error(log.With(c.logger, "metric", metric.Name)).Log("log", "failed to get prometheus metric", "err", err)
+				level.Error(log.With(mgs.logger, "metric", metric.Name)).Log("log", "failed to get prometheus metric", "err", err)
 			} else {
 				proMetrics <- m
 			}
 		}
-
 	}
-	c.metrics.Collect(proMetrics)
 }
 
 type Collects []CollectConfig

collector/labels.go

@@ -87,6 +87,19 @@ func (re *Regexp) UnmarshalYAML(value *yaml.Node) error {
 	}
 	return nil
 }
+func (re *Regexp) UnmarshalJSON(raw []byte) error {
+	var regex string
+	if err := json.Unmarshal(raw, &regex); err != nil {
+		return err
+	} else if len(regex) != 0 {
+		if r, err := NewRegexp(regex); err != nil {
+			return err
+		} else {
+			*re = r
+		}
+	}
+	return nil
+}
 
 type Action string
 
@@ -112,7 +125,7 @@ const (
 type RelabelConfig struct {
 	// A list of labels from which values are taken and concatenated
 	// with the configured separator in order.
-	SourceLabels model.LabelNames `yaml:"source_labels,flow,omitempty"`
+	SourceLabels model.LabelNames `yaml:"source_labels,flow,omitempty" json:"source_labels"`
 	// Separator is the string between concatenated values from the source labels.
 	Separator string `yaml:"separator,omitempty"`
 	// Regex against which the concatenation is matched.
@@ -123,7 +136,7 @@ type RelabelConfig struct {
 	Modulus uint64 `yaml:"modulus,omitempty"`
 	// TargetLabel is the label to which the resulting string is written in a replacement.
 	// Regexp interpolation is allowed for the replace action.
-	TargetLabel string `yaml:"target_label,omitempty"`
+	TargetLabel string `yaml:"target_label,omitempty" json:"target_label"`
 	// Replacement is the regex replacement pattern to be used.
 	Replacement string `yaml:"replacement,omitempty"`
 	// Action is the action to be performed for the relabeling.
@@ -150,12 +163,7 @@ func (rcs RelabelConfigs) String() string {
 	return string(marshal)
 }
 
-func (c *RelabelConfig) UnmarshalYAML(value *yaml.Node) error {
-	*c = DefaultRelabelConfig
-	type plain RelabelConfig
-	if err := value.Decode((*plain)(c)); err != nil {
-		return err
-	}
+func (c *RelabelConfig) init() error {
 	if c.Regex.Regexp == nil {
 		c.Regex = MustNewRegexp("")
 	}
@@ -190,9 +198,24 @@ func (c *RelabelConfig) UnmarshalYAML(value *yaml.Node) error {
 			return fmt.Errorf("%s action requires only 'regex', and no other fields", c.Action)
 		}
 	}
-
 	return nil
+}
 
+func (c *RelabelConfig) UnmarshalJSON(raw []byte) error {
+	*c = DefaultRelabelConfig
+	type plain RelabelConfig
+	if err := json.Unmarshal(raw, (*plain)(c)); err != nil {
+		return err
+	}
+	return c.init()
+}
+func (c *RelabelConfig) UnmarshalYAML(value *yaml.Node) error {
+	*c = DefaultRelabelConfig
+	type plain RelabelConfig
+	if err := value.Decode((*plain)(c)); err != nil {
+		return err
+	}
+	return c.init()
 }
 
 // Process returns a relabeled copy of the given label set. The relabel configurations