From 690b2863ae4698e7de58a9d8da49137a2d6fb806 Mon Sep 17 00:00:00 2001 From: Alan Sandoval Date: Tue, 13 May 2025 11:09:18 -0500 Subject: [PATCH 1/8] add new field to config --- pkg/config/config.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 21b5d74..de44263 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -146,10 +146,11 @@ type MetricConfig struct { ForceMonotonicy bool `yaml:"force_monotonicy"` ConstantLabels map[string]string `yaml:"const_labels"` DynamicLabels map[string]string `yaml:"dynamic_labels"` + InheritLabels []string `yaml:"inherit_labels"` StringValueMapping *StringValueMappingConfig `yaml:"string_value_mapping"` MQTTValueScale float64 `yaml:"mqtt_value_scale"` // ErrorValue is used while error during value parsing - ErrorValue *float64 `yaml:"error_value"` + ErrorValue *float64 `yaml:"error_value"` } // StringValueMappingConfig defines the mapping from string to float @@ -162,6 +163,8 @@ type StringValueMappingConfig struct { func (mc *MetricConfig) PrometheusDescription() *prometheus.Desc { labels := append([]string{"sensor", "topic"}, mc.DynamicLabelsKeys()...) + labels = append(labels, mc.InheritLabels...) + fmt.Println("labels: ", labels) return prometheus.NewDesc( mc.PrometheusName, mc.Help, labels, mc.ConstantLabels, ) @@ -257,7 +260,7 @@ func LoadConfig(configFile string, logger *zap.Logger) (Config, error) { logger.Warn("string_value_mapping.error_value is deprecated: please use error_value at the metric level.", zap.String("prometheusName", m.PrometheusName), zap.String("MQTTName", m.MQTTName)) } - if m.Expression != "" && m.RawExpression != "" { + if m.Expression != "" && m.RawExpression != "" { return Config{}, fmt.Errorf("metric %s/%s: expression and raw_expression are mutually exclusive.", m.MQTTName, m.PrometheusName) } } From b5d3d692db32085f96d589913345447c74048c08 Mon Sep 17 00:00:00 2001 From: Alan Sandoval Date: Tue, 13 May 2025 15:07:44 -0500 Subject: [PATCH 2/8] seems to work lol --- .dockerignore | 3 ++- config.yaml.dist | 2 ++ hack/docker-compose.yml | 18 ++++++------- pkg/config/config.go | 1 - pkg/metrics/collector.go | 2 ++ pkg/metrics/extractor.go | 12 ++++++++- pkg/metrics/parser.go | 56 ++++++++++++++++++++++++++++++++++++++-- 7 files changed, 80 insertions(+), 14 deletions(-) diff --git a/.dockerignore b/.dockerignore index 1470bf4..364e960 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,4 @@ bin/ .git -systemd/ \ No newline at end of file +systemd/ +hack diff --git a/config.yaml.dist b/config.yaml.dist index ef4c0f8..e6959da 100644 --- a/config.yaml.dist +++ b/config.yaml.dist @@ -46,6 +46,8 @@ metrics: # A map of string to string for constant labels. This labels will be attached to every prometheus metric const_labels: sensor_type: dht22 + inherit_labels: + - serialNumber # The name of the metric in prometheus - prom_name: humidity # The name of the metric in a MQTT JSON message diff --git a/hack/docker-compose.yml b/hack/docker-compose.yml index beee3f2..99f9ffd 100644 --- a/hack/docker-compose.yml +++ b/hack/docker-compose.yml @@ -14,18 +14,18 @@ services: - 9641:9641 volumes: - type: bind - source: ./${CONFIG:-dht22.yaml} + source: ./${CONFIG:-config.yaml} target: /config.yaml mosquitto: image: eclipse-mosquitto:1.6.15 ports: - 1883:1883 - 9001:9001 - prometheus: - image: prom/prometheus:v2.55.1 - ports: - - 9090:9090 - volumes: - - type: bind - source: ./prometheus.yml - target: /etc/prometheus/prometheus.yml + # prometheus: + # image: prom/prometheus:v2.55.1 + # ports: + # - 9090:9090 + # volumes: + # - type: bind + # source: ./prometheus.yml + # target: /etc/prometheus/prometheus.yml diff --git a/pkg/config/config.go b/pkg/config/config.go index de44263..6d26af2 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -164,7 +164,6 @@ type StringValueMappingConfig struct { func (mc *MetricConfig) PrometheusDescription() *prometheus.Desc { labels := append([]string{"sensor", "topic"}, mc.DynamicLabelsKeys()...) labels = append(labels, mc.InheritLabels...) - fmt.Println("labels: ", labels) return prometheus.NewDesc( mc.PrometheusName, mc.Help, labels, mc.ConstantLabels, ) diff --git a/pkg/metrics/collector.go b/pkg/metrics/collector.go index 61fcc40..3717f8c 100644 --- a/pkg/metrics/collector.go +++ b/pkg/metrics/collector.go @@ -80,6 +80,8 @@ func (c *MemoryCachedCollector) Collect(mc chan<- prometheus.Metric) { labels = append(labels, metric.Labels[k]) } + fmt.Println("Prometheus description", metric.Description) + m := prometheus.MustNewConstMetric( metric.Description, metric.ValueType, diff --git a/pkg/metrics/extractor.go b/pkg/metrics/extractor.go index d528c78..4add357 100644 --- a/pkg/metrics/extractor.go +++ b/pkg/metrics/extractor.go @@ -24,6 +24,7 @@ func NewJSONObjectExtractor(p Parser) Extractor { return func(topic string, payload []byte, deviceID string) (MetricCollection, error) { var mc MetricCollection parsed := gojsonq.New(gojsonq.SetSeparator(p.separator)).FromString(string(payload)) + rawPayload := gojsonq.New(gojsonq.SetSeparator(p.separator)).FromString(string(payload)) for path := range p.config() { rawValue := parsed.Find(path) @@ -33,16 +34,25 @@ func NewJSONObjectExtractor(p Parser) Extractor { } // Find all valid metric configs + // var labels map[string]string for _, config := range p.findMetricConfigs(path, deviceID) { id := metricID(topic, path, deviceID, config.PrometheusName) m, err := p.parseMetric(config, id, rawValue) if err != nil { - return nil, fmt.Errorf("failed to parse valid value from '%v' for metric %q: %w", rawValue, config.PrometheusName, err) + return nil, fmt.Errorf("failed to parse valid json value from '%v' for metric %q: %w", rawValue, config.PrometheusName, err) } + labels, err := p.parseInheritedLabels(config, m, rawPayload) + if err != nil { + return nil, fmt.Errorf("failed to parse valid json labels from '%v' for metric %q: %w", rawPayload, config.PrometheusName, err) + } + m.Labels = labels + m.LabelsKeys = append(m.LabelsKeys, config.InheritLabels...) m.Topic = topic mc = append(mc, m) + fmt.Println("extracted?", m) } } + fmt.Println("final mc?", mc) return mc, nil } } diff --git a/pkg/metrics/parser.go b/pkg/metrics/parser.go index f7493c8..2331249 100644 --- a/pkg/metrics/parser.go +++ b/pkg/metrics/parser.go @@ -12,6 +12,7 @@ import ( "github.com/expr-lang/expr" "github.com/expr-lang/expr/vm" "github.com/hikhvar/mqtt2prometheus/pkg/config" + "github.com/thedevsaddam/gojsonq/v2" "gopkg.in/yaml.v2" ) @@ -153,6 +154,17 @@ func NewParser(metrics []config.MetricConfig, separator, stateDir string) Parser for i := range metrics { key := metrics[i].MQTTName cfgs[key] = append(cfgs[key], &metrics[i]) + // for j := range metrics[i].InheritLabels { + // label := metrics[i].InheritLabels[j] + // cfgs[key] = append(cfgs[label], &metrics[i]) + // } + // } + // fmt.Println("created a new parser config: ", cfgs) + // for k, v := range cfgs { + // fmt.Println(k, "value is", v) + // for n, p := range v { + // fmt.Println(n, "nested value is", p) + // } } return Parser{ separator: separator, @@ -178,6 +190,31 @@ func (p *Parser) findMetricConfigs(metric string, deviceID string) []*config.Met return configs } +// parseInheritedLabels parses the given JSON data and extracts the listed labels +// to append them to the Prometheus Metric +// this function returns a map of labels +func (p *Parser) parseInheritedLabels(cfg *config.MetricConfig, m Metric, payloadJson *gojsonq.JSONQ) (map[string]string, error) { + + // includes already-defined labels that are provided by parseMetric() + labels := m.Labels + + // inherit labels + if len(cfg.InheritLabels) > 0 { + var jsonCopy *gojsonq.JSONQ + for _, v := range cfg.InheritLabels { + jsonCopy = payloadJson.Copy() + result, err := jsonCopy.From(v).GetR() + if err != nil { + return labels, fmt.Errorf("failed to parse labels from '%v' for label %q: %w", jsonCopy, v, err) + } + this_label, _ := result.String() + labels[v] = this_label + } + } + fmt.Println("result labels", labels) + return labels, nil +} + // parseMetric parses the given value according to the given deviceID and metricPath. The config allows to // parse a metric value according to the device ID. func (p *Parser) parseMetric(cfg *config.MetricConfig, metricID string, value interface{}) (Metric, error) { @@ -185,6 +222,7 @@ func (p *Parser) parseMetric(cfg *config.MetricConfig, metricID string, value in var err error if cfg.RawExpression != "" { + fmt.Println("parsing as cfg.RawExpression = nil") if metricValue, err = p.evalExpressionValue(metricID, cfg.RawExpression, value, metricValue); err != nil { if cfg.ErrorValue != nil { metricValue = *cfg.ErrorValue @@ -195,6 +233,7 @@ func (p *Parser) parseMetric(cfg *config.MetricConfig, metricID string, value in } else { if boolValue, ok := value.(bool); ok { + fmt.Println("parsing as bool") if boolValue { metricValue = 1 } else { @@ -204,12 +243,13 @@ func (p *Parser) parseMetric(cfg *config.MetricConfig, metricID string, value in // If string value mapping is defined, use that if cfg.StringValueMapping != nil { + fmt.Println("parsing as StringValueMapping") floatValue, ok := cfg.StringValueMapping.Map[strValue] if ok { metricValue = floatValue - // deprecated, replaced by ErrorValue from the upper level + // deprecated, replaced by ErrorValue from the upper level } else if cfg.StringValueMapping.ErrorValue != nil { metricValue = *cfg.StringValueMapping.ErrorValue } else if cfg.ErrorValue != nil { @@ -219,6 +259,7 @@ func (p *Parser) parseMetric(cfg *config.MetricConfig, metricID string, value in } } else { + fmt.Println("Parsing as float") // otherwise try to parse float floatValue, err := strconv.ParseFloat(strValue, 64) @@ -235,14 +276,17 @@ func (p *Parser) parseMetric(cfg *config.MetricConfig, metricID string, value in } } else if floatValue, ok := value.(float64); ok { + fmt.Println("Not parsing, setting as float64") metricValue = floatValue } else if cfg.ErrorValue != nil { + fmt.Println("parsing as ErrorValue") metricValue = *cfg.ErrorValue } else { return Metric{}, fmt.Errorf("got data with unexpectd type: %T ('%v')", value, value) } if cfg.Expression != "" { + fmt.Println("parsing as Expression") if metricValue, err = p.evalExpressionValue(metricID, cfg.Expression, value, metricValue); err != nil { if cfg.ErrorValue != nil { metricValue = *cfg.ErrorValue @@ -272,8 +316,13 @@ func (p *Parser) parseMetric(cfg *config.MetricConfig, metricID string, value in ingestTime = now() } - // generate dynamic labels + // build labels var labels map[string]string + if len(cfg.DynamicLabels) > 0 || len(cfg.InheritLabels) > 0 { + labels = make(map[string]string, len(cfg.DynamicLabels)+len(cfg.InheritLabels)) + } + + // generate dynamic labels if len(cfg.DynamicLabels) > 0 { labels = make(map[string]string, len(cfg.DynamicLabels)) for k, v := range cfg.DynamicLabels { @@ -332,6 +381,7 @@ func (p *Parser) writeMetricState(metricID string, state *metricState) error { if err != nil { return err } + fmt.Println("writeMetricState opening file...") f, err := os.OpenFile(p.stateFileName(metricID), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { return err @@ -349,6 +399,7 @@ func (p *Parser) getMetricState(metricID string) (*metricState, error) { var err error state, found := p.states[metricID] if !found { + fmt.Println("Parser getMetricState not found") if state, err = p.readMetricState(metricID); err != nil { return nil, err } @@ -356,6 +407,7 @@ func (p *Parser) getMetricState(metricID string) (*metricState, error) { } // Write the state back to disc every minute. if now().Sub(state.lastWritten) >= time.Minute { + fmt.Println("Parser attempting to write") if err = p.writeMetricState(metricID, state); err == nil { state.lastWritten = now() } From d85984bde3a8f9561e414f4f4346cfc5d4af552e Mon Sep 17 00:00:00 2001 From: Alan Sandoval Date: Tue, 13 May 2025 15:18:07 -0500 Subject: [PATCH 3/8] revert hack/docker-compose.yaml --- hack/docker-compose.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/hack/docker-compose.yml b/hack/docker-compose.yml index 99f9ffd..beee3f2 100644 --- a/hack/docker-compose.yml +++ b/hack/docker-compose.yml @@ -14,18 +14,18 @@ services: - 9641:9641 volumes: - type: bind - source: ./${CONFIG:-config.yaml} + source: ./${CONFIG:-dht22.yaml} target: /config.yaml mosquitto: image: eclipse-mosquitto:1.6.15 ports: - 1883:1883 - 9001:9001 - # prometheus: - # image: prom/prometheus:v2.55.1 - # ports: - # - 9090:9090 - # volumes: - # - type: bind - # source: ./prometheus.yml - # target: /etc/prometheus/prometheus.yml + prometheus: + image: prom/prometheus:v2.55.1 + ports: + - 9090:9090 + volumes: + - type: bind + source: ./prometheus.yml + target: /etc/prometheus/prometheus.yml From 48225c5b56d3a5abc42b9864428bacb5ab24af5c Mon Sep 17 00:00:00 2001 From: Alan Sandoval Date: Tue, 13 May 2025 15:19:33 -0500 Subject: [PATCH 4/8] rm debug print statements --- pkg/metrics/collector.go | 2 -- pkg/metrics/parser.go | 22 ---------------------- 2 files changed, 24 deletions(-) diff --git a/pkg/metrics/collector.go b/pkg/metrics/collector.go index 3717f8c..61fcc40 100644 --- a/pkg/metrics/collector.go +++ b/pkg/metrics/collector.go @@ -80,8 +80,6 @@ func (c *MemoryCachedCollector) Collect(mc chan<- prometheus.Metric) { labels = append(labels, metric.Labels[k]) } - fmt.Println("Prometheus description", metric.Description) - m := prometheus.MustNewConstMetric( metric.Description, metric.ValueType, diff --git a/pkg/metrics/parser.go b/pkg/metrics/parser.go index 2331249..7754f2f 100644 --- a/pkg/metrics/parser.go +++ b/pkg/metrics/parser.go @@ -154,17 +154,6 @@ func NewParser(metrics []config.MetricConfig, separator, stateDir string) Parser for i := range metrics { key := metrics[i].MQTTName cfgs[key] = append(cfgs[key], &metrics[i]) - // for j := range metrics[i].InheritLabels { - // label := metrics[i].InheritLabels[j] - // cfgs[key] = append(cfgs[label], &metrics[i]) - // } - // } - // fmt.Println("created a new parser config: ", cfgs) - // for k, v := range cfgs { - // fmt.Println(k, "value is", v) - // for n, p := range v { - // fmt.Println(n, "nested value is", p) - // } } return Parser{ separator: separator, @@ -211,7 +200,6 @@ func (p *Parser) parseInheritedLabels(cfg *config.MetricConfig, m Metric, payloa labels[v] = this_label } } - fmt.Println("result labels", labels) return labels, nil } @@ -222,7 +210,6 @@ func (p *Parser) parseMetric(cfg *config.MetricConfig, metricID string, value in var err error if cfg.RawExpression != "" { - fmt.Println("parsing as cfg.RawExpression = nil") if metricValue, err = p.evalExpressionValue(metricID, cfg.RawExpression, value, metricValue); err != nil { if cfg.ErrorValue != nil { metricValue = *cfg.ErrorValue @@ -233,7 +220,6 @@ func (p *Parser) parseMetric(cfg *config.MetricConfig, metricID string, value in } else { if boolValue, ok := value.(bool); ok { - fmt.Println("parsing as bool") if boolValue { metricValue = 1 } else { @@ -243,7 +229,6 @@ func (p *Parser) parseMetric(cfg *config.MetricConfig, metricID string, value in // If string value mapping is defined, use that if cfg.StringValueMapping != nil { - fmt.Println("parsing as StringValueMapping") floatValue, ok := cfg.StringValueMapping.Map[strValue] if ok { @@ -259,7 +244,6 @@ func (p *Parser) parseMetric(cfg *config.MetricConfig, metricID string, value in } } else { - fmt.Println("Parsing as float") // otherwise try to parse float floatValue, err := strconv.ParseFloat(strValue, 64) @@ -276,17 +260,14 @@ func (p *Parser) parseMetric(cfg *config.MetricConfig, metricID string, value in } } else if floatValue, ok := value.(float64); ok { - fmt.Println("Not parsing, setting as float64") metricValue = floatValue } else if cfg.ErrorValue != nil { - fmt.Println("parsing as ErrorValue") metricValue = *cfg.ErrorValue } else { return Metric{}, fmt.Errorf("got data with unexpectd type: %T ('%v')", value, value) } if cfg.Expression != "" { - fmt.Println("parsing as Expression") if metricValue, err = p.evalExpressionValue(metricID, cfg.Expression, value, metricValue); err != nil { if cfg.ErrorValue != nil { metricValue = *cfg.ErrorValue @@ -381,7 +362,6 @@ func (p *Parser) writeMetricState(metricID string, state *metricState) error { if err != nil { return err } - fmt.Println("writeMetricState opening file...") f, err := os.OpenFile(p.stateFileName(metricID), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { return err @@ -399,7 +379,6 @@ func (p *Parser) getMetricState(metricID string) (*metricState, error) { var err error state, found := p.states[metricID] if !found { - fmt.Println("Parser getMetricState not found") if state, err = p.readMetricState(metricID); err != nil { return nil, err } @@ -407,7 +386,6 @@ func (p *Parser) getMetricState(metricID string) (*metricState, error) { } // Write the state back to disc every minute. if now().Sub(state.lastWritten) >= time.Minute { - fmt.Println("Parser attempting to write") if err = p.writeMetricState(metricID, state); err == nil { state.lastWritten = now() } From d2a0ba81ea311940eb36a2e86781420f0e68dc4f Mon Sep 17 00:00:00 2001 From: Alan Sandoval Date: Tue, 13 May 2025 15:20:38 -0500 Subject: [PATCH 5/8] fmt --- pkg/config/config.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 6d26af2..aea6e5b 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -149,8 +149,7 @@ type MetricConfig struct { InheritLabels []string `yaml:"inherit_labels"` StringValueMapping *StringValueMappingConfig `yaml:"string_value_mapping"` MQTTValueScale float64 `yaml:"mqtt_value_scale"` - // ErrorValue is used while error during value parsing - ErrorValue *float64 `yaml:"error_value"` + ErrorValue *float64 `yaml:"error_value"` // ErrorValue is used while error during value parsing } // StringValueMappingConfig defines the mapping from string to float From 4231fa7fdbc6eb0d84179730d4e4aefa7a5fc086 Mon Sep 17 00:00:00 2001 From: Alan Sandoval Date: Tue, 13 May 2025 15:48:55 -0500 Subject: [PATCH 6/8] cross-platform build & rm last dangling print statements --- Dockerfile | 4 ++++ pkg/metrics/extractor.go | 2 -- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 267ee47..8112432 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,9 @@ FROM golang:1.23 as builder +ENV CGO_ENABLED=1 +ENV GOOS=linux +ENV GOARCH=amd64 + COPY . /build/mqtt2prometheus WORKDIR /build/mqtt2prometheus RUN make static_build TARGET_FILE=/bin/mqtt2prometheus diff --git a/pkg/metrics/extractor.go b/pkg/metrics/extractor.go index 4add357..35cad52 100644 --- a/pkg/metrics/extractor.go +++ b/pkg/metrics/extractor.go @@ -49,10 +49,8 @@ func NewJSONObjectExtractor(p Parser) Extractor { m.LabelsKeys = append(m.LabelsKeys, config.InheritLabels...) m.Topic = topic mc = append(mc, m) - fmt.Println("extracted?", m) } } - fmt.Println("final mc?", mc) return mc, nil } } From 3b460aa847aafde9b7d3e3484acdc2b1f4f0fa61 Mon Sep 17 00:00:00 2001 From: Alan Sandoval Date: Tue, 13 May 2025 15:53:29 -0500 Subject: [PATCH 7/8] wut build fails now --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8112432..f4fe240 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.23 as builder +FROM golang:1.24 AS builder ENV CGO_ENABLED=1 ENV GOOS=linux From 1159d595aac18ce7bbfb7a94a99b68d59af8f653 Mon Sep 17 00:00:00 2001 From: Alan Sandoval Date: Tue, 13 May 2025 15:57:18 -0500 Subject: [PATCH 8/8] some notes --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index f4fe240..d358f93 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,7 @@ FROM golang:1.24 AS builder +# enable cross-platform builds with CGO_ENABLED +# I had to first compile without buildx for buildx to then work ENV CGO_ENABLED=1 ENV GOOS=linux ENV GOARCH=amd64