diff --git a/.dockerignore b/.dockerignore index 1470bf4..364e960 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,4 @@ bin/ .git -systemd/ \ No newline at end of file +systemd/ +hack diff --git a/Dockerfile b/Dockerfile index 267ee47..d358f93 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,10 @@ -FROM golang:1.23 as builder +FROM golang:1.24 AS builder + +# enable cross-platform builds with CGO_ENABLED +# I had to first compile without buildx for buildx to then work +ENV CGO_ENABLED=1 +ENV GOOS=linux +ENV GOARCH=amd64 COPY . /build/mqtt2prometheus WORKDIR /build/mqtt2prometheus diff --git a/config.yaml.dist b/config.yaml.dist index ef4c0f8..e6959da 100644 --- a/config.yaml.dist +++ b/config.yaml.dist @@ -46,6 +46,8 @@ metrics: # A map of string to string for constant labels. This labels will be attached to every prometheus metric const_labels: sensor_type: dht22 + inherit_labels: + - serialNumber # The name of the metric in prometheus - prom_name: humidity # The name of the metric in a MQTT JSON message diff --git a/pkg/config/config.go b/pkg/config/config.go index 21b5d74..aea6e5b 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -146,10 +146,10 @@ type MetricConfig struct { ForceMonotonicy bool `yaml:"force_monotonicy"` ConstantLabels map[string]string `yaml:"const_labels"` DynamicLabels map[string]string `yaml:"dynamic_labels"` + InheritLabels []string `yaml:"inherit_labels"` StringValueMapping *StringValueMappingConfig `yaml:"string_value_mapping"` MQTTValueScale float64 `yaml:"mqtt_value_scale"` - // ErrorValue is used while error during value parsing - ErrorValue *float64 `yaml:"error_value"` + ErrorValue *float64 `yaml:"error_value"` // ErrorValue is used while error during value parsing } // StringValueMappingConfig defines the mapping from string to float @@ -162,6 +162,7 @@ type StringValueMappingConfig struct { func (mc *MetricConfig) PrometheusDescription() *prometheus.Desc { labels := append([]string{"sensor", "topic"}, mc.DynamicLabelsKeys()...) + labels = append(labels, mc.InheritLabels...) return prometheus.NewDesc( mc.PrometheusName, mc.Help, labels, mc.ConstantLabels, ) @@ -257,7 +258,7 @@ func LoadConfig(configFile string, logger *zap.Logger) (Config, error) { logger.Warn("string_value_mapping.error_value is deprecated: please use error_value at the metric level.", zap.String("prometheusName", m.PrometheusName), zap.String("MQTTName", m.MQTTName)) } - if m.Expression != "" && m.RawExpression != "" { + if m.Expression != "" && m.RawExpression != "" { return Config{}, fmt.Errorf("metric %s/%s: expression and raw_expression are mutually exclusive.", m.MQTTName, m.PrometheusName) } } diff --git a/pkg/metrics/extractor.go b/pkg/metrics/extractor.go index d528c78..35cad52 100644 --- a/pkg/metrics/extractor.go +++ b/pkg/metrics/extractor.go @@ -24,6 +24,7 @@ func NewJSONObjectExtractor(p Parser) Extractor { return func(topic string, payload []byte, deviceID string) (MetricCollection, error) { var mc MetricCollection parsed := gojsonq.New(gojsonq.SetSeparator(p.separator)).FromString(string(payload)) + rawPayload := gojsonq.New(gojsonq.SetSeparator(p.separator)).FromString(string(payload)) for path := range p.config() { rawValue := parsed.Find(path) @@ -33,12 +34,19 @@ func NewJSONObjectExtractor(p Parser) Extractor { } // Find all valid metric configs + // var labels map[string]string for _, config := range p.findMetricConfigs(path, deviceID) { id := metricID(topic, path, deviceID, config.PrometheusName) m, err := p.parseMetric(config, id, rawValue) if err != nil { - return nil, fmt.Errorf("failed to parse valid value from '%v' for metric %q: %w", rawValue, config.PrometheusName, err) + return nil, fmt.Errorf("failed to parse valid json value from '%v' for metric %q: %w", rawValue, config.PrometheusName, err) } + labels, err := p.parseInheritedLabels(config, m, rawPayload) + if err != nil { + return nil, fmt.Errorf("failed to parse valid json labels from '%v' for metric %q: %w", rawPayload, config.PrometheusName, err) + } + m.Labels = labels + m.LabelsKeys = append(m.LabelsKeys, config.InheritLabels...) m.Topic = topic mc = append(mc, m) } diff --git a/pkg/metrics/parser.go b/pkg/metrics/parser.go index f7493c8..7754f2f 100644 --- a/pkg/metrics/parser.go +++ b/pkg/metrics/parser.go @@ -12,6 +12,7 @@ import ( "github.com/expr-lang/expr" "github.com/expr-lang/expr/vm" "github.com/hikhvar/mqtt2prometheus/pkg/config" + "github.com/thedevsaddam/gojsonq/v2" "gopkg.in/yaml.v2" ) @@ -178,6 +179,30 @@ func (p *Parser) findMetricConfigs(metric string, deviceID string) []*config.Met return configs } +// parseInheritedLabels parses the given JSON data and extracts the listed labels +// to append them to the Prometheus Metric +// this function returns a map of labels +func (p *Parser) parseInheritedLabels(cfg *config.MetricConfig, m Metric, payloadJson *gojsonq.JSONQ) (map[string]string, error) { + + // includes already-defined labels that are provided by parseMetric() + labels := m.Labels + + // inherit labels + if len(cfg.InheritLabels) > 0 { + var jsonCopy *gojsonq.JSONQ + for _, v := range cfg.InheritLabels { + jsonCopy = payloadJson.Copy() + result, err := jsonCopy.From(v).GetR() + if err != nil { + return labels, fmt.Errorf("failed to parse labels from '%v' for label %q: %w", jsonCopy, v, err) + } + this_label, _ := result.String() + labels[v] = this_label + } + } + return labels, nil +} + // parseMetric parses the given value according to the given deviceID and metricPath. The config allows to // parse a metric value according to the device ID. func (p *Parser) parseMetric(cfg *config.MetricConfig, metricID string, value interface{}) (Metric, error) { @@ -209,7 +234,7 @@ func (p *Parser) parseMetric(cfg *config.MetricConfig, metricID string, value in if ok { metricValue = floatValue - // deprecated, replaced by ErrorValue from the upper level + // deprecated, replaced by ErrorValue from the upper level } else if cfg.StringValueMapping.ErrorValue != nil { metricValue = *cfg.StringValueMapping.ErrorValue } else if cfg.ErrorValue != nil { @@ -272,8 +297,13 @@ func (p *Parser) parseMetric(cfg *config.MetricConfig, metricID string, value in ingestTime = now() } - // generate dynamic labels + // build labels var labels map[string]string + if len(cfg.DynamicLabels) > 0 || len(cfg.InheritLabels) > 0 { + labels = make(map[string]string, len(cfg.DynamicLabels)+len(cfg.InheritLabels)) + } + + // generate dynamic labels if len(cfg.DynamicLabels) > 0 { labels = make(map[string]string, len(cfg.DynamicLabels)) for k, v := range cfg.DynamicLabels {