jvm-observ-lib: Add java_micrometer_with_suffixes option

There is yet another metric pattern combinatino. When micrometer metrics are received through OTEL bridge then prometheus suffixes might not be added. Adding this option to jvm/process lib

Author: v-zhuravlev

jvm-observ-lib/README.md

@@ -7,7 +7,8 @@ Supports the following sources:
 - `prometheus` (https://prometheus.github.io/client_java/instrumentation/jvm/#jvm-memory-metrics). This also works for jmx_exporter (javaagent mode) starting from 1.0.1 release.
 - `otel` (https://github.com/open-telemetry/opentelemetry-java-contrib/blob/main/jmx-metrics/docs/target-systems/jvm.md)
 - `otel_with_suffixes` same as otel with add_metric_suffixes=true in otelcollector
-- `java_micrometer` (springboot) (https://github.com/micrometer-metrics/micrometer/blob/main/micrometer-core/src/main/java/io/micrometer/core/instrument/binder/jvm/JvmMemoryMetrics.java)
+- `java_micrometer` (springboot) (https://github.com/micrometer-metrics/micrometer/blob/main/micrometer-core/src/main/java/io/micrometer/core/instrument/binder/jvm/JvmMemoryMetrics.java). Can be seen when Micrometer to OTEL bridge is used.
+- `java_micrometer_with_suffixes` (springboot) same, but with prometheus suffixes
 - `prometheus_old` client_java instrumentation prior to 1.0.0 release: (https://github.com/prometheus/client_java/releases/tag/v1.0.0-alpha-4). This also works for jmx_exporter (javaagent mode) prior to 1.0.1 release.
 - `jmx_exporter`. Works with jmx_exporter (both http and javaagent modes) and the folllowing snippet:
 
@@ -45,7 +46,7 @@ local jvm =
         uid: 'jvm-sample',
         dashboardNamePrefix: 'JVM',
         dashboardTags: ['java', 'jvm'],
-        metricsSource: 'java_micrometer', // or java_otel, prometheus,
+        metricsSource: 'java_micrometer_with_suffixes', // or java_otel, prometheus,
     }
   );
 jvm.asMonitoringMixin()

jvm-observ-lib/main.libsonnet

@@ -15,7 +15,7 @@ local processlib = import 'process-observ-lib/main.libsonnet';
           filteringSelector: this.config.filteringSelector,
           groupLabels: this.config.groupLabels,
           instanceLabels: this.config.instanceLabels,
-          uid: this.config.uid,
+          uid: this.config.uid - '-jvm',
           dashboardNamePrefix: this.config.dashboardNamePrefix,
           dashboardTags: this.config.dashboardTags,
           metricsSource:
@@ -25,6 +25,7 @@ local processlib = import 'process-observ-lib/main.libsonnet';
             + (if std.member(this.config.metricsSource, 'prometheus') then ['prometheus'] else [])
             + (if std.member(this.config.metricsSource, 'jmx_exporter') then ['jmx_exporter'] else [])
             + (if std.member(this.config.metricsSource, 'prometheus_old') then ['prometheus'] else [])
+            + (if std.member(this.config.metricsSource, 'java_micrometer_with_suffixes') then ['java_micrometer_with_suffixes'] else [])
             + (if std.member(this.config.metricsSource, 'java_micrometer') then ['java_micrometer'] else []),
         }
       ),

jvm-observ-lib/signals/buffers.libsonnet

@@ -8,10 +8,11 @@ function(this)
     aggLevel: 'group',
     aggFunction: 'avg',
     discoveryMetric: {
-      java_micrometer: 'jvm_buffer_memory_used_bytes',  // https://github.com/micrometer-metrics/micrometer/blob/main/micrometer-core/src/main/java/io/micrometer/core/instrument/binder/jvm/JvmMemoryMetrics.java
-      prometheus: 'jvm_buffer_pool_used_bytes',  // https://prometheus.github.io/client_java/instrumentation/jvm/#jvm-buffer-pool-metrics
+      java_micrometer: 'jvm_buffer_memory_used',  // can be seen when otel micrometer bridge is used
+      java_micrometer_with_suffixes: 'jvm_buffer_memory_used_bytes',  // https://github.com/micrometer-metrics/micrometer/blob/main/micrometer-core/src/main/java/io/micrometer/core/instrument/binder/jvm/JvmMemoryMetrics.java
       otel: 'process_runtime_jvm_buffer_usage',
       otel_with_suffixes: 'process_runtime_jvm_buffer_usage_bytes',
+      prometheus: 'jvm_buffer_pool_used_bytes',  // https://prometheus.github.io/client_java/instrumentation/jvm/#jvm-buffer-pool-metrics
       prometheus_old: 'jvm_buffer_pool_used_bytes',
     },
     signals: {
@@ -23,6 +24,9 @@ function(this)
         optional: true,
         sources: {
           java_micrometer: {
+            expr: 'jvm_buffer_memory_used{%(queriesSelector)s}',
+          },
+          java_micrometer_with_suffixes: {
             expr: 'jvm_buffer_memory_used_bytes{%(queriesSelector)s}',
           },
           prometheus: {
@@ -47,6 +51,9 @@ function(this)
         optional: true,
         sources: {
           java_micrometer: {
+            expr: 'jvm_buffer_total_capacity{%(queriesSelector)s}',
+          },
+          java_micrometer_with_suffixes: {
             expr: 'jvm_buffer_total_capacity_bytes{%(queriesSelector)s}',
           },
           prometheus: {
@@ -71,6 +78,9 @@ function(this)
         optional: true,
         sources: {
           java_micrometer: {
+            expr: 'jvm_buffer_memory_used{%(queriesSelector)s}',
+          },
+          java_micrometer_with_suffixes: {
             expr: 'jvm_buffer_memory_used_bytes{%(queriesSelector)s}',
           },
           prometheus: {
@@ -95,6 +105,9 @@ function(this)
         optional: true,
         sources: {
           java_micrometer: {
+            expr: 'jvm_buffer_total_capacity{%(queriesSelector)s}',
+          },
+          java_micrometer_with_suffixes: {
             expr: 'jvm_buffer_total_capacity_bytes{%(queriesSelector)s}',
           },
           prometheus: {

jvm-observ-lib/signals/classes.libsonnet

@@ -8,7 +8,8 @@ function(this)
     aggLevel: 'group',
     aggFunction: 'avg',
     discoveryMetric: {
-      java_micrometer: 'jvm_classes_loaded_classes',  // https://github.com/micrometer-metrics/micrometer/blob/main/micrometer-core/src/main/java/io/micrometer/core/instrument/binder/jvm/ClassLoaderMetrics.java
+      java_micrometer: 'jvm_classes_loaded',  // https://github.com/micrometer-metrics/micrometer/blob/main/micrometer-core/src/main/java/io/micrometer/core/instrument/binder/jvm/ClassLoaderMetrics.java
+      java_micrometer_with_suffixes: 'jvm_classes_loaded_classes',
       prometheus: 'jvm_classes_loaded',  // https://prometheus.github.io/client_java/instrumentation/jvm/#jvm-class-loading-metrics
       otel: 'process_runtime_jvm_classes_loaded',
       otel_with_suffixes: 'process_runtime_jvm_classes_loaded_total',
@@ -23,6 +24,9 @@ function(this)
         unit: 'short',
         sources: {
           java_micrometer: {
+            expr: 'jvm_classes_loaded{%(queriesSelector)s}',
+          },
+          java_micrometer_with_suffixes: {
             expr: 'jvm_classes_loaded_classes{%(queriesSelector)s}',
           },
           prometheus: {

jvm-observ-lib/signals/gc.libsonnet

@@ -6,7 +6,8 @@ function(this)
     aggLevel: 'group',
     aggFunction: 'avg',
     discoveryMetric: {
-      java_micrometer: 'jvm_memory_used_bytes',
+      java_micrometer: 'jvm_memory_used',
+      java_micrometer_with_suffixes: 'jvm_memory_used_bytes',
       prometheus: 'jvm_memory_used_bytes',  // https://prometheus.github.io/client_java/instrumentation/jvm/#jvm-memory-metrics
       prometheus_old: 'jvm_memory_bytes_used',
       otel: 'process_runtime_jvm_memory_usage',  //https://opentelemetry.io/docs/specs/semconv/runtime/jvm-metrics/
@@ -28,6 +29,14 @@ function(this)
         optional: true,
         sources: {
           java_micrometer: {
+            expr: 'jvm_gc_pause_count{%(queriesSelector)s}',
+            aggKeepLabels: ['action', 'cause'],
+            legendCustomTemplate: '%(aggLegend)s',
+            exprWrappers: [
+              actionLabelPrettify,
+            ],
+          },
+          java_micrometer_with_suffixes: {
             expr: 'jvm_gc_pause_seconds_count{%(queriesSelector)s}',
             aggKeepLabels: ['action', 'cause'],
             legendCustomTemplate: '%(aggLegend)s',
@@ -63,6 +72,13 @@ function(this)
         optional: true,
         sources: {
           java_micrometer: {
+            expr: 'avg by (action,cause, %(agg)s) (rate(jvm_gc_pause_max{%(queriesSelector)s}[$__rate_interval]))',
+            legendCustomTemplate: '{{ %(agg)s }}: {{ action }} ({{ cause }}) (max)',
+            exprWrappers: [
+              actionLabelPrettify,
+            ],
+          },
+          java_micrometer_with_suffixes: {
             expr: 'avg by (action,cause, %(agg)s) (rate(jvm_gc_pause_seconds_max{%(queriesSelector)s}[$__rate_interval]))',
             legendCustomTemplate: '{{ %(agg)s }}: {{ action }} ({{ cause }}) (max)',
             exprWrappers: [
@@ -79,6 +95,19 @@ function(this)
         optional: true,
         sources: {
           java_micrometer: {
+            expr: |||
+              avg by (%(agg)s, action, cause) (
+                rate(jvm_gc_pause_sum{%(queriesSelector)s}[$__rate_interval])
+                /
+                rate(jvm_gc_pause_count{%(queriesSelector)s}[$__rate_interval])
+              )
+            |||,
+            legendCustomTemplate: '{{ %(agg)s }}: {{ action }} ({{ cause }}) (avg)',
+            exprWrappers: [
+              actionLabelPrettify,
+            ],
+          },
+          java_micrometer_with_suffixes: {
             expr: |||
               avg by (%(agg)s, action, cause) (
                 rate(jvm_gc_pause_seconds_sum{%(queriesSelector)s}[$__rate_interval])
@@ -138,6 +167,9 @@ function(this)
         optional: true,
         sources: {
           java_micrometer: {
+            expr: 'jvm_gc_memory_promoted{%(queriesSelector)s}',
+          },
+          java_micrometer_with_suffixes: {
             expr: 'jvm_gc_memory_promoted_bytes_total{%(queriesSelector)s}',
           },
         },
@@ -151,6 +183,9 @@ function(this)
         optional: true,
         sources: {
           java_micrometer: {
+            expr: 'jvm_gc_memory_allocated{%(queriesSelector)s}',
+          },
+          java_micrometer_with_suffixes: {
             expr: 'jvm_gc_memory_allocated_bytes_total{%(queriesSelector)s}',
           },
           prometheus: {
@@ -192,6 +227,9 @@ function(this)
           java_micrometer: {
             expr: 'jvm_memory_used_bytes{id=~"(G1 |PS )?Eden Space", area="heap", %(queriesSelector)s}',
           },
+          java_micrometer_with_suffixes: {
+            expr: 'jvm_memory_used{id=~"(G1 |PS )?Eden Space", area="heap", %(queriesSelector)s}',
+          },
           prometheus: {
             expr: 'jvm_memory_pool_used_bytes{pool=~"(G1 |PS )?Eden Space", %(queriesSelector)s}',
           },
@@ -223,6 +261,12 @@ function(this)
         optional: true,
         sources: {
           java_micrometer: {
+            expr: 'jvm_memory_max{id=~"(G1 )?Eden Space", area="heap", %(queriesSelector)s}',
+            exprWrappers: [
+              ['', ' != -1'],
+            ],
+          },
+          java_micrometer_with_suffixes: {
             expr: 'jvm_memory_max_bytes{id=~"(G1 )?Eden Space", area="heap", %(queriesSelector)s}',
             exprWrappers: [
               ['', ' != -1'],
@@ -255,6 +299,9 @@ function(this)
         optional: true,
         sources: {
           java_micrometer: {
+            expr: 'jvm_memory_committed{id=~"(G1 |PS )?Eden Space", area="heap", %(queriesSelector)s}',
+          },
+          java_micrometer_with_suffixes: {
             expr: 'jvm_memory_committed_bytes{id=~"(G1 |PS )?Eden Space", area="heap", %(queriesSelector)s}',
           },
           prometheus: {
@@ -283,6 +330,9 @@ function(this)
         sources: {
           //spring
           java_micrometer: {
+            expr: 'jvm_memory_used{id=~"(G1 |PS )?Survivor Space", area="heap", %(queriesSelector)s}',
+          },
+          java_micrometer_with_suffixes: {
             expr: 'jvm_memory_used_bytes{id=~"(G1 |PS )?Survivor Space", area="heap", %(queriesSelector)s}',
           },
           prometheus: {
@@ -316,6 +366,9 @@ function(this)
         optional: true,
         sources: {
           java_micrometer: {
+            expr: 'jvm_memory_max{id=~"(G1 |PS )?Survivor Space", area="heap", %(queriesSelector)s} != -1',
+          },
+          java_micrometer_with_suffixes: {
             expr: 'jvm_memory_max_bytes{id=~"(G1 |PS )?Survivor Space", area="heap", %(queriesSelector)s} != -1',
           },
           prometheus: {
@@ -342,6 +395,9 @@ function(this)
         optional: true,
         sources: {
           java_micrometer: {
+            expr: 'jvm_memory_committed{id=~"(G1 |PS )?Survivor Space", area="heap", %(queriesSelector)s}',
+          },
+          java_micrometer_with_suffixes: {
             expr: 'jvm_memory_committed_bytes{id=~"(G1 |PS )?Survivor Space", area="heap", %(queriesSelector)s}',
           },
           prometheus: {
@@ -369,6 +425,9 @@ function(this)
         sources: {
           //spring
           java_micrometer: {
+            expr: 'jvm_memory_used{id="Tenured Gen", area="heap", %(queriesSelector)s}',
+          },
+          java_micrometer_with_suffixes: {
             expr: 'jvm_memory_used_bytes{id="Tenured Gen", area="heap", %(queriesSelector)s}',
           },
           prometheus: {
@@ -402,6 +461,12 @@ function(this)
         optional: true,
         sources: {
           java_micrometer: {
+            expr: 'jvm_memory_max{id="Tenured Gen", area="heap", %(queriesSelector)s}',
+            exprWrappers: [
+              ['', ' != -1'],
+            ],
+          },
+          java_micrometer_with_suffixes: {
             expr: 'jvm_memory_max_bytes{id="Tenured Gen", area="heap", %(queriesSelector)s}',
             exprWrappers: [
               ['', ' != -1'],
@@ -440,6 +505,9 @@ function(this)
         optional: true,
         sources: {
           java_micrometer: {
+            expr: 'jvm_memory_committed{id="Tenured Gen", area="heap", %(queriesSelector)s}',
+          },
+          java_micrometer_with_suffixes: {
             expr: 'jvm_memory_committed_bytes{id="Tenured Gen", area="heap", %(queriesSelector)s}',
           },
           prometheus: {

jvm-observ-lib/signals/hikari.libsonnet

@@ -24,6 +24,7 @@ function(this)
             expr: 'hikaricp_connections{%(queriesSelector)s}',
             aggKeepLabels: ['pool'],
           },
+          java_micrometer_with_suffixes: self.java_micrometer,
           otel: {
             expr: 'hikaricp_connections{%(queriesSelector)s}',
             aggKeepLabels: ['pool'],
@@ -42,6 +43,10 @@ function(this)
             expr: 'hikaricp_connections_timeout_total{%(queriesSelector)s}',
             aggKeepLabels: ['pool'],
           },
+          java_micrometer_with_suffixes: {
+            expr: 'hikaricp_connections_timeout{%(queriesSelector)s}',
+            aggKeepLabels: ['pool'],
+          },
         },
       },
       connectionsActive: {
@@ -55,6 +60,7 @@ function(this)
             expr: 'hikaricp_connections_active{%(queriesSelector)s}',
             aggKeepLabels: ['pool'],
           },
+          java_micrometer_with_suffixes: self.java_micrometer,
           otel: {
             expr: 'hikaricp_connections_active{%(queriesSelector)s}',
             aggKeepLabels: ['pool'],
@@ -73,6 +79,7 @@ function(this)
             expr: 'hikaricp_connections_idle{%(queriesSelector)s}',
             aggKeepLabels: ['pool'],
           },
+          java_micrometer_with_suffixes: self.java_micrometer,
           otel: {
             expr: 'hikaricp_connections_idle{%(queriesSelector)s}',
             aggKeepLabels: ['pool'],
@@ -91,6 +98,7 @@ function(this)
             expr: 'hikaricp_connections_pending{%(queriesSelector)s}',
             aggKeepLabels: ['pool'],
           },
+          java_micrometer_with_suffixes: self.java_micrometer,
           otel: {
             expr: 'hikaricp_connections_pending{%(queriesSelector)s}',
             aggKeepLabels: ['pool'],
@@ -107,12 +115,20 @@ function(this)
         optional: true,
         sources: {
           java_micrometer: {
+            expr: |||
+              rate(hikaricp_connections_creation_sum{%(queriesSelector)s}[$__rate_interval])
+              /rate(hikaricp_connections_creation_count{%(queriesSelector)s}[$__rate_interval])
+            |||,
+            aggKeepLabels: ['pool'],
+          },
+          java_micrometer_with_suffixes: {
             expr: |||
               rate(hikaricp_connections_creation_seconds_sum{%(queriesSelector)s}[$__rate_interval])
               /rate(hikaricp_connections_creation_seconds_count{%(queriesSelector)s}[$__rate_interval])
             |||,
             aggKeepLabels: ['pool'],
           },
+
         },
       },
       connectionsUsageDurationAvg: {
@@ -123,6 +139,13 @@ function(this)
         optional: true,
         sources: {
           java_micrometer: {
+            expr: |||
+              rate(hikaricp_connections_usage_sum{%(queriesSelector)s}[$__rate_interval])
+              /rate(hikaricp_connections_usage_count{%(queriesSelector)s}[$__rate_interval])
+            |||,
+            aggKeepLabels: ['pool'],
+          },
+          java_micrometer_with_suffixes: {
             expr: |||
               rate(hikaricp_connections_usage_seconds_sum{%(queriesSelector)s}[$__rate_interval])
               /rate(hikaricp_connections_usage_seconds_count{%(queriesSelector)s}[$__rate_interval])
@@ -139,6 +162,13 @@ function(this)
         optional: true,
         sources: {
           java_micrometer: {
+            expr: |||
+              rate(hikaricp_connections_acquire_sum{%(queriesSelector)s}[$__rate_interval])
+              /rate(hikaricp_connections_acquire_count{%(queriesSelector)s}[$__rate_interval])
+            |||,
+            aggKeepLabels: ['pool'],
+          },
+          java_micrometer_with_suffixes: {
             expr: |||
               rate(hikaricp_connections_acquire_seconds_sum{%(queriesSelector)s}[$__rate_interval])
               /rate(hikaricp_connections_acquire_seconds_count{%(queriesSelector)s}[$__rate_interval])