[improve] http protocol prometheus parsing optimization (apache#1996)

Co-authored-by: zhangshenghang <[email protected]>

Author: zhangshenghang

collector/src/main/java/org/apache/hertzbeat/collector/collect/http/HttpCollectImpl.java

@@ -97,7 +97,7 @@ public class HttpCollectImpl extends AbstractCollect {
     public static final String OpenAIUsageAPI = "/dashboard/billing/usage";
     public static final String startDate = "start_date";
     public static final String endDate = "end_date";
-
+    private static final Map<Long, ExporterParser> EXPORTER_PARSER_TABLE = new ConcurrentHashMap<>();
     private final Set<Integer> defaultSuccessStatusCodes = Stream.of(HttpStatus.SC_OK, HttpStatus.SC_CREATED,
             HttpStatus.SC_ACCEPTED, HttpStatus.SC_MULTIPLE_CHOICES, HttpStatus.SC_MOVED_PERMANENTLY,
             HttpStatus.SC_MOVED_TEMPORARILY).collect(Collectors.toSet());
@@ -367,8 +367,6 @@ private void parseResponseByPromQl(String resp, List<String> aliasFields, HttpPr
         prometheusParser.handle(resp, aliasFields, http, builder);
     }
 
-    private static final Map<Long, ExporterParser> EXPORTER_PARSER_TABLE = new ConcurrentHashMap<>();
-
     private void parseResponseByPrometheusExporter(String resp, List<String> aliasFields,
                                                    CollectRep.MetricsData.Builder builder) {
         if (!EXPORTER_PARSER_TABLE.containsKey(builder.getId())) {
@@ -381,8 +379,8 @@ private void parseResponseByPrometheusExporter(String resp, List<String> aliasFi
             MetricFamily metricFamily = metricFamilyMap.get(metrics);
             for (MetricFamily.Metric metric : metricFamily.getMetricList()) {
                 Map<String, String> labelMap = metric.getLabelPair()
-                                                       .stream()
-                                                       .collect(Collectors.toMap(MetricFamily.Label::getName, MetricFamily.Label::getValue));
+                        .stream()
+                        .collect(Collectors.toMap(MetricFamily.Label::getName, MetricFamily.Label::getValue));
                 CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
                 for (String aliasField : aliasFields) {
                     if ("value".equals(aliasField)) {
@@ -394,9 +392,14 @@ private void parseResponseByPrometheusExporter(String resp, List<String> aliasFi
                             valueRowBuilder.addColumns(String.valueOf(metric.getUntyped().getValue()));
                         } else if (metric.getInfo() != null) {
                             valueRowBuilder.addColumns(String.valueOf(metric.getInfo().getValue()));
+                        } else if (metric.getSummary() != null) {
+                            valueRowBuilder.addColumns(String.valueOf(metric.getSummary().getValue()));
+                        } else if (metric.getHistogram() != null) {
+                            valueRowBuilder.addColumns(String.valueOf(metric.getHistogram().getValue()));
                         }
                     } else {
-                        valueRowBuilder.addColumns(labelMap.get(aliasField));
+                        String columnValue = labelMap.get(aliasField);
+                        valueRowBuilder.addColumns(columnValue == null ? CommonConstants.NULL_VALUE : columnValue);
                     }
                 }
                 builder.addValues(valueRowBuilder.build());
@@ -443,6 +446,7 @@ private void getValueFromJson(List<String> aliasFields, CollectRep.MetricsData.B
 
     /**
      * create httpContext
+     *
      * @param httpProtocol http protocol
      * @return context
      */
@@ -451,7 +455,7 @@ public HttpContext createHttpContext(HttpProtocol httpProtocol) {
         if (auth != null && DispatchConstants.DIGEST_AUTH.equals(auth.getType())) {
             HttpClientContext clientContext = new HttpClientContext();
             if (StringUtils.hasText(auth.getDigestAuthUsername())
-                        && StringUtils.hasText(auth.getDigestAuthPassword())) {
+                    && StringUtils.hasText(auth.getDigestAuthPassword())) {
                 CredentialsProvider provider = new BasicCredentialsProvider();
                 UsernamePasswordCredentials credentials = new UsernamePasswordCredentials(auth.getDigestAuthUsername(),
                         auth.getDigestAuthPassword());
@@ -468,6 +472,7 @@ public HttpContext createHttpContext(HttpProtocol httpProtocol) {
 
     /**
      * create http request
+     *
      * @param httpProtocol http params
      * @return http uri request
      */
@@ -522,7 +527,7 @@ public HttpUriRequest createHttpRequest(HttpProtocol httpProtocol) {
         }
         // add accept
         if (DispatchConstants.PARSE_DEFAULT.equals(httpProtocol.getParseType())
-                    || DispatchConstants.PARSE_JSON_PATH.equals(httpProtocol.getParseType())) {
+                || DispatchConstants.PARSE_JSON_PATH.equals(httpProtocol.getParseType())) {
             requestBuilder.addHeader(HttpHeaders.ACCEPT, "application/json");
         } else if (DispatchConstants.PARSE_XML_PATH.equals(httpProtocol.getParseType())) {
             requestBuilder.addHeader(HttpHeaders.ACCEPT, "text/xml,application/xml");
@@ -537,7 +542,7 @@ public HttpUriRequest createHttpRequest(HttpProtocol httpProtocol) {
                 requestBuilder.addHeader(HttpHeaders.AUTHORIZATION, value);
             } else if (DispatchConstants.BASIC_AUTH.equals(authorization.getType())) {
                 if (StringUtils.hasText(authorization.getBasicAuthUsername())
-                            && StringUtils.hasText(authorization.getBasicAuthPassword())) {
+                        && StringUtils.hasText(authorization.getBasicAuthPassword())) {
                     String authStr = authorization.getBasicAuthUsername() + ":" + authorization.getBasicAuthPassword();
                     String encodedAuth = new String(Base64.encodeBase64(authStr.getBytes(StandardCharsets.UTF_8)), StandardCharsets.UTF_8);
                     requestBuilder.addHeader(HttpHeaders.AUTHORIZATION, DispatchConstants.BASIC + " " + encodedAuth);
@@ -558,8 +563,8 @@ public HttpUriRequest createHttpRequest(HttpProtocol httpProtocol) {
         } else {
             String ipAddressType = IpDomainUtil.checkIpAddressType(httpProtocol.getHost());
             String baseUri = CollectorConstants.IPV6.equals(ipAddressType)
-                                     ? String.format("[%s]:%s%s", httpProtocol.getHost(), httpProtocol.getPort(), uri)
-                                     : String.format("%s:%s%s", httpProtocol.getHost(), httpProtocol.getPort(), uri);
+                    ? String.format("[%s]:%s%s", httpProtocol.getHost(), httpProtocol.getPort(), uri)
+                    : String.format("%s:%s%s", httpProtocol.getHost(), httpProtocol.getPort(), uri);
             boolean ssl = Boolean.parseBoolean(httpProtocol.getSsl());
             if (ssl) {
                 requestBuilder.setUri(CollectorConstants.HTTPS_HEADER + baseUri);
@@ -572,10 +577,10 @@ public HttpUriRequest createHttpRequest(HttpProtocol httpProtocol) {
         int timeout = CollectUtil.getTimeout(httpProtocol.getTimeout(), 0);
         if (timeout > 0) {
             RequestConfig requestConfig = RequestConfig.custom()
-                                                  .setConnectTimeout(timeout)
-                                                  .setSocketTimeout(timeout)
-                                                  .setRedirectsEnabled(true)
-                                                  .build();
+                    .setConnectTimeout(timeout)
+                    .setSocketTimeout(timeout)
+                    .setRedirectsEnabled(true)
+                    .build();
             requestBuilder.setConfig(requestConfig);
         }
         return requestBuilder.build();

collector/src/main/java/org/apache/hertzbeat/collector/collect/http/promethus/exporter/ExporterParser.java

@@ -27,7 +27,6 @@
 import lombok.extern.slf4j.Slf4j;
 import org.apache.hertzbeat.collector.collect.http.promethus.ParseException;
 import org.apache.hertzbeat.common.util.StrBuffer;
-import org.springframework.util.StringUtils;
 
 /**
  * Resolves the data passed by prometheus's exporter interface http:xxx/metrics
@@ -38,7 +37,7 @@ public class ExporterParser {
     private static final String HELP = "HELP";
     private static final String TYPE = "TYPE";
     private static final String EOF = "EOF";
-
+    private static final String METRIC_NAME_LABEL = ".name";
     private static final String QUANTILE_LABEL = "quantile";
     private static final String BUCKET_LABEL = "le";
     private static final String NAME_LABEL = "__name__";
@@ -52,13 +51,11 @@ public class ExporterParser {
     private static final char ENTER = '\n';
     private static final char SPACE = ' ';
     private static final char COMMA = ',';
-
+    private final Lock lock = new ReentrantLock();
     private MetricFamily currentMetricFamily;
     private String currentQuantile;
     private String currentBucket;
 
-    private final Lock lock = new ReentrantLock();
-
     public Map<String, MetricFamily> textToMetric(String resp) {
         // key: metric name, value: metric family
         Map<String, MetricFamily> metricMap = new ConcurrentHashMap<>(10);
@@ -86,7 +83,8 @@ private void parseLine(Map<String, MetricFamily> metricMap, StrBuffer buffer) {
                 this.currentMetricFamily = null;
                 this.parseComment(metricMap, buffer);
             }
-            case ENTER -> {}
+            case ENTER -> {
+            }
             default -> {
                 this.currentBucket = null;
                 this.currentQuantile = null;
@@ -114,7 +112,8 @@ private void parseComment(Map<String, MetricFamily> metricMap, StrBuffer buffer)
         switch (token) {
             case HELP -> this.parseHelp(buffer);
             case TYPE -> this.parseType(buffer);
-            default -> {}
+            default -> {
+            }
         }
     }
 
@@ -135,28 +134,27 @@ private void parseType(StrBuffer line) {
 
     private void parseMetric(StrBuffer buffer) {
         String metricName = this.readTokenAsMetricName(buffer);
+        MetricFamily.Label label = new MetricFamily.Label();
+        label.setName(METRIC_NAME_LABEL);
+        label.setValue(metricName);
+
         if (metricName.isEmpty()) {
             log.error("error parse metric, metric name is null, line: {}", buffer.toStr());
             return;
         }
+
         List<MetricFamily.Metric> metricList = this.currentMetricFamily.getMetricList();
         if (metricList == null) {
             metricList = new ArrayList<>();
             this.currentMetricFamily.setMetricList(metricList);
         }
-        // TODO: This part may have issues. The current logic creates only one metric for both HISTOGRAM and SUMMARY
-        // compared to the source code, there is a slight modification: the source code stores parsing results in a property
-        // here, the results are passed through parameters.
-        MetricFamily.Metric metric;
-        if (!metricList.isEmpty()
-                && (this.currentMetricFamily.getMetricType().equals(MetricType.HISTOGRAM)
-                || this.currentMetricFamily.getMetricType().equals(MetricType.SUMMARY))) {
-            metric = metricList.get(0);
-        } else {
-            metric = new MetricFamily.Metric();
-            metricList.add(metric);
-        }
 
+        // TODO For the time being, the data is displayed in the form of labels. If there is a better chart display method in the future, we will optimize it.
+        MetricFamily.Metric metric = new MetricFamily.Metric();
+        metricList.add(metric);
+
+        metric.setLabelPair(new ArrayList<>());
+        metric.getLabelPair().add(label);
         this.readLabels(metric, buffer);
     }
 
@@ -165,7 +163,6 @@ private void readLabels(MetricFamily.Metric metric, StrBuffer buffer) {
         if (buffer.isEmpty()) {
             return;
         }
-        metric.setLabelPair(new ArrayList<>());
         if (buffer.charAt(0) == LEFT_CURLY_BRACKET) {
             buffer.read();
             this.startReadLabelName(metric, buffer);
@@ -218,9 +215,9 @@ private void startReadLabelValue(MetricFamily.Metric metric, MetricFamily.Label
             this.currentQuantile = labelValue;
         } else if (this.currentMetricFamily.getMetricType().equals(MetricType.HISTOGRAM) && label.getName().equals(BUCKET_LABEL)) {
             this.currentBucket = labelValue;
-        } else {
-            metric.getLabelPair().add(label);
         }
+        metric.getLabelPair().add(label);
+
         if (buffer.isEmpty()) {
             return;
         }
@@ -257,47 +254,16 @@ private void readLabelValue(MetricFamily.Metric metric, MetricFamily.Label label
                 metric.setUntyped(untyped);
             }
             case SUMMARY -> {
-                MetricFamily.Summary summary = metric.getSummary();
-                if (summary == null) {
-                    summary = new MetricFamily.Summary();
-                    metric.setSummary(summary);
-                }
-                // Process data for xxx_sum
-                if (label != null && this.isSum(label.getName())) {
-                    summary.setSum(buffer.toDouble());
-                }
-                // Process data for xxx_count
-                else if (label != null && this.isCount(label.getName())) {
-                    summary.setCount(buffer.toLong());
-                }
-                // Handle format for "xxx{quantile=\"0\"} 0"
-                else if (StringUtils.hasText(this.currentQuantile)) {
-                    List<MetricFamily.Quantile> quantileList = summary.getQuantileList();
-                    MetricFamily.Quantile quantile = new MetricFamily.Quantile();
-                    quantile.setXLabel(StrBuffer.parseDouble(this.currentQuantile));
-                    quantile.setValue(buffer.toDouble());
-                    quantileList.add(quantile);
-                }
+                // For the time being, the data is displayed in the form of labels. If there is a better chart display method in the future, we will optimize it.
+                MetricFamily.Summary summary = new MetricFamily.Summary();
+                summary.setValue(buffer.toDouble());
+                metric.setSummary(summary);
             }
             case HISTOGRAM -> {
-                MetricFamily.Histogram histogram = metric.getHistogram();
-                if (histogram == null) {
-                    histogram = new MetricFamily.Histogram();
-                    metric.setHistogram(histogram);
-                }
-                if (label != null && this.isSum(label.getName())) {
-                    histogram.setSum(buffer.toDouble());
-                } else if (label != null && this.isCount(label.getName())) {
-                    histogram.setCount(buffer.toLong());
-                }
-                // Process the format "xxx{quantile=\"0\"} 0"
-                else if (StringUtils.hasText(this.currentBucket)) {
-                    List<MetricFamily.Bucket> bucketList = histogram.getBucketList();
-                    MetricFamily.Bucket bucket = new MetricFamily.Bucket();
-                    bucket.setUpperBound(StrBuffer.parseDouble(this.currentBucket));
-                    bucket.setCumulativeCount(buffer.toLong());
-                    bucketList.add(bucket);
-                }
+                // For the time being, the data is displayed in the form of labels. If there is a better chart display method in the future, we will optimize it.
+                MetricFamily.Histogram histogram = new MetricFamily.Histogram();
+                histogram.setValue(buffer.toDouble());
+                metric.setHistogram(histogram);
             }
             default -> throw new ParseException("no such type in metricFamily");
         }
@@ -390,7 +356,9 @@ private String readTokenAsLabelValue(StrBuffer buffer) {
                 escaped = false;
             } else {
                 switch (c) {
-                    case QUOTES -> { return builder.toString(); }
+                    case QUOTES -> {
+                        return builder.toString();
+                    }
                     case ENTER -> throw new ParseException("parse label value error, next line");
                     case '\\' -> escaped = true;
                     default -> builder.append(c);

collector/src/main/java/org/apache/hertzbeat/collector/collect/http/promethus/exporter/MetricFamily.java

@@ -169,6 +169,11 @@ public static class Untyped {
     @Data
     public static class Summary {
 
+        /**
+         * value
+         */
+        private double value;
+
         /**
          * count
          */
@@ -207,6 +212,11 @@ public static class Quantile {
     @Data
     public static class Histogram {
 
+        /**
+         * value
+         */
+        private double value;
+
         /**
          * count
          */

collector/src/main/java/org/apache/hertzbeat/collector/dispatch/MetricsCollect.java

@@ -264,7 +264,7 @@ private void calculateFields(Metrics metrics, CollectRep.MetricsData.Builder col
                                 break;
                             }
                         }
-                        // valueList为空时也执行,涵盖纯字符串赋值表达式
+                        // Also executed when valueList is empty, covering pure string assignment expressions
                         Object objValue = JexlExpressionRunner.evaluate(expression, fieldValueMap);
                         if (objValue != null) {
                             value = String.valueOf(objValue);