diff --git a/jvm-observ-lib/README.md b/jvm-observ-lib/README.md index 4dbb6c4ae..ad28032ed 100644 --- a/jvm-observ-lib/README.md +++ b/jvm-observ-lib/README.md @@ -1,15 +1,17 @@ -# JVM observability lib +# JVM observability library -This lib can be used to generate dashboards, rows, panels, and alerts for JVM monitoring. +This library can be used to generate dashboards, rows, panels, and alerts for JVM monitoring. Supports the following sources: - `prometheus` (https://prometheus.github.io/client_java/instrumentation/jvm/#jvm-memory-metrics). This also works for jmx_exporter (javaagent mode) starting from 1.0.1 release. -- `otel` (https://github.com/open-telemetry/opentelemetry-java-contrib/blob/main/jmx-metrics/docs/target-systems/jvm.md) -- `otel_with_suffixes` same as otel with add_metric_suffixes=true in otelcollector -- `java_micrometer` (springboot) (https://github.com/micrometer-metrics/micrometer/blob/main/micrometer-core/src/main/java/io/micrometer/core/instrument/binder/jvm/JvmMemoryMetrics.java) +- `otel_with_suffixes` OpenTelemetry JVM metrics following semantic conventions (https://opentelemetry.io/docs/specs/semconv/runtime/jvm-metrics/). Uses metric names like `jvm_memory_used_bytes`, `jvm_gc_duration_seconds`, etc. **Recommended for new deployments. Corresponds to semantic conventions v1.22.0 and later.** +- `otel_old` Legacy OpenTelemetry JVM metrics (https://github.com/open-telemetry/opentelemetry-java-contrib/blob/main/jmx-metrics/docs/target-systems/jvm.md). Uses `process_runtime_jvm_*` metric names. **Corresponds to semantic conventions v1.21.0 and earlier.** +- `otel_old_with_suffixes` Same as `otel_old` but with Prometheus suffixes added by the OTel collector (`add_metric_suffixes=true`). **Corresponds to semantic conventions v1.21.0 and earlier.** +- `java_micrometer` (springboot) (https://github.com/micrometer-metrics/micrometer/blob/main/micrometer-core/src/main/java/io/micrometer/core/instrument/binder/jvm/JvmMemoryMetrics.java). Can be seen when Micrometer to OTEL bridge is used. +- `java_micrometer_with_suffixes` (springboot) same, but with prometheus suffixes - `prometheus_old` client_java instrumentation prior to 1.0.0 release: (https://github.com/prometheus/client_java/releases/tag/v1.0.0-alpha-4). This also works for jmx_exporter (javaagent mode) prior to 1.0.1 release. -- `jmx_exporter`. Works with jmx_exporter (both http and javaagent modes) and the folllowing snippet: +- `jmx_exporter`. Works with jmx_exporter (both http and javaagent modes) and the following snippet: ```yaml lowercaseOutputName: true @@ -22,6 +24,43 @@ rules: - pattern : java.lang ``` +## OpenTelemetry sources + +The library supports multiple OpenTelemetry metric sources to accommodate different deployment scenarios: + +### Recommended: `otel_with_suffixes` +Uses the latest OpenTelemetry JVM semantic conventions with metric names like: +- `jvm_memory_used_bytes{jvm_memory_type="heap", jvm_memory_pool_name="G1 Eden Space"}` +- `jvm_gc_duration_seconds_count{jvm_gc_name="G1 Young Generation", jvm_gc_action="end of minor GC"}` +- `jvm_thread_count{jvm_thread_daemon="true", jvm_thread_state="runnable"}` + +### Legacy: `otel_old` and `otel_old_with_suffixes` +For backward compatibility with existing OpenTelemetry deployments using: +- `process_runtime_jvm_memory_usage{type="heap", pool="G1 Eden Space"}` +- `process_runtime_jvm_gc_duration_count{gc="G1 Young Generation", action="end of minor GC"}` +- `process_runtime_jvm_threads_count{daemon="true"}` + +### Migration path +1. **Current users of `otel`** → migrate to `otel_old` (no functional change) +2. **Current users of `otel_with_suffixes`** → migrate to `otel_old_with_suffixes` (no functional change) +3. **New deployments** → use `otel_with_suffixes` for semantic conventions compliance +4. **Upgrade path** → `otel_old*` → `otel_with_suffixes` when updating instrumentation + +## OpenTelemetry Semantic Conventions Migration + +The OpenTelemetry project introduced breaking changes to JVM metric names in semantic conventions v1.22.0 (October 2023): + +- **v1.21.0 and earlier**: Used `process_runtime_jvm_*` metric names (supported by `otel_old` sources) +- **v1.22.0 and later**: Changed to `jvm.*` metric names (supported by `otel_with_suffixes` sources) + +### When to use each source: + +- Use `otel_old*` sources if your instrumentation produces `process_runtime_jvm_*` metrics +- Use `otel_with_suffixes` if your instrumentation produces `jvm.*` metrics +- Check your OpenTelemetry Java instrumentation version: + - **Before semantic conventions v1.22.0**: Use `otel_old*` sources + - **After semantic conventions v1.22.0**: Use `otel_with_suffixes` sources + ## Import @@ -45,7 +84,7 @@ local jvm = uid: 'jvm-sample', dashboardNamePrefix: 'JVM', dashboardTags: ['java', 'jvm'], - metricsSource: 'java_micrometer', // or java_otel, prometheus, + metricsSource: 'otel_with_suffixes', // or java_micrometer_with_suffixes, prometheus, etc. } ); jvm.asMonitoringMixin() diff --git a/jvm-observ-lib/config.libsonnet b/jvm-observ-lib/config.libsonnet index 5f265a1f1..ee148d422 100644 --- a/jvm-observ-lib/config.libsonnet +++ b/jvm-observ-lib/config.libsonnet @@ -6,7 +6,8 @@ uid: 'jvm', dashboardNamePrefix: '', dashboardTags: ['java', 'jvm'], - metricsSource: ['java_micrometer', 'prometheus', 'prometheus_old', 'otel', 'otel_with_suffixes', 'jmx_exporter'], + // generate only popular by default. See others options in README. + metricsSource: ['java_micrometer', 'prometheus', 'otel_old_with_suffixes', 'otel_with_suffixes', 'jmx_exporter'], signals+: { memory: (import './signals/memory.libsonnet')(this), diff --git a/jvm-observ-lib/dashboards_out/jvm-dashboard.json b/jvm-observ-lib/dashboards_out/jvm-dashboard.json index c05868b44..bfd8507d7 100644 --- a/jvm-observ-lib/dashboards_out/jvm-dashboard.json +++ b/jvm-observ-lib/dashboards_out/jvm-dashboard.json @@ -51,7 +51,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "((avg by (job) (\n java_lang_memory_heapmemoryusage_used{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_bytes_used{area=\"heap\", job=~\"$job\",instance=~\"$instance\"})\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_used_bytes{area=\"heap\", job=~\"$job\",instance=~\"$instance\"})\n)\nor\navg by (job) (\n sum without (pool) (process_runtime_jvm_memory_usage_bytes{type=\"heap\", job=~\"$job\",instance=~\"$instance\"})\n)\nor\navg by (job) (\n sum without (pool) (process_runtime_jvm_memory_usage{type=\"heap\", job=~\"$job\",instance=~\"$instance\"})\n))/(avg by (job) (\n java_lang_memory_heapmemoryusage_max{job=~\"$job\",instance=~\"$instance\"} != -1\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_bytes_max{area=\"heap\", job=~\"$job\",instance=~\"$instance\"} != -1)\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_max_bytes{area=\"heap\", job=~\"$job\",instance=~\"$instance\"} != -1)\n)\nor\navg by (job) (\n sum without (pool) (process_runtime_jvm_memory_limit_bytes{type=\"heap\", job=~\"$job\",instance=~\"$instance\"} != -1)\n)\nor\navg by (job) (\n sum without (pool) (process_runtime_jvm_memory_limit{type=\"heap\", job=~\"$job\",instance=~\"$instance\"} != -1)\n))) * 100", + "expr": "((avg by (job) (\n java_lang_memory_heapmemoryusage_used{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_used_bytes{area=\"heap\", job=~\"$job\",instance=~\"$instance\"})\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_used{area=\"heap\", job=~\"$job\",instance=~\"$instance\"})\n)\nor\navg by (job) (\n sum without (jvm_memory_pool_name) (jvm_memory_used_bytes{jvm_memory_type=\"heap\", job=~\"$job\",instance=~\"$instance\"})\n)\nor\navg by (job) (\n sum without (pool) (process_runtime_jvm_memory_usage_bytes{type=\"heap\", job=~\"$job\",instance=~\"$instance\"})\n))/(avg by (job) (\n java_lang_memory_heapmemoryusage_max{job=~\"$job\",instance=~\"$instance\"} != -1\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_max_bytes{area=\"heap\", job=~\"$job\",instance=~\"$instance\"} != -1)\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_max{area=\"heap\", job=~\"$job\",instance=~\"$instance\"} != -1)\n)\nor\navg by (job) (\n sum without (jvm_memory_pool_name) (jvm_memory_limit_bytes{jvm_memory_type=\"heap\", job=~\"$job\",instance=~\"$instance\"} != -1)\n)\nor\navg by (job) (\n sum without (pool) (process_runtime_jvm_memory_limit_bytes{type=\"heap\", job=~\"$job\",instance=~\"$instance\"} != -1)\n))) * 100", "format": "time_series", "instant": false, "legendFormat": "{{job}}: JVM memory used(heap)", @@ -100,7 +100,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "((avg by (job) (\n java_lang_memory_nonheapmemoryusage_used{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_bytes_used{area=\"nonheap\", job=~\"$job\",instance=~\"$instance\"})\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_used_bytes{area=\"nonheap\", job=~\"$job\",instance=~\"$instance\"})\n)\nor\navg by (job) (\n sum without (pool) (process_runtime_jvm_memory_usage_bytes{type=\"non_heap\", job=~\"$job\",instance=~\"$instance\"})\n)\nor\navg by (job) (\n sum without (pool) (process_runtime_jvm_memory_usage{type=\"non_heap\", job=~\"$job\",instance=~\"$instance\"})\n))/(avg by (job) (\n java_lang_memory_nonheapmemoryusage_max{job=~\"$job\",instance=~\"$instance\"} != -1\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_bytes_max{area=\"nonheap\", job=~\"$job\",instance=~\"$instance\"} != -1)\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_max_bytes{area=\"nonheap\", job=~\"$job\",instance=~\"$instance\"} != -1)\n)\nor\navg by (job) (\n sum without (pool) (process_runtime_jvm_memory_limit_bytes{type=\"non_heap\", job=~\"$job\",instance=~\"$instance\"} != -1)\n)\nor\navg by (job) (\n sum without (pool) (process_runtime_jvm_memory_limit{type=\"non_heap\", job=~\"$job\",instance=~\"$instance\"} != -1)\n))) * 100", + "expr": "((avg by (job) (\n java_lang_memory_nonheapmemoryusage_used{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_used_bytes{area=\"nonheap\", job=~\"$job\",instance=~\"$instance\"})\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_used{area=\"nonheap\", job=~\"$job\",instance=~\"$instance\"})\n)\nor\navg by (job) (\n sum without (jvm_memory_pool_name) (jvm_memory_used_bytes{jvm_memory_type=\"non_heap\", job=~\"$job\",instance=~\"$instance\"})\n)\nor\navg by (job) (\n sum without (pool) (process_runtime_jvm_memory_usage_bytes{type=\"non_heap\", job=~\"$job\",instance=~\"$instance\"})\n))/(avg by (job) (\n java_lang_memory_nonheapmemoryusage_max{job=~\"$job\",instance=~\"$instance\"} != -1\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_max_bytes{area=\"nonheap\", job=~\"$job\",instance=~\"$instance\"} != -1)\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_max{area=\"nonheap\", job=~\"$job\",instance=~\"$instance\"} != -1)\n)\nor\navg by (job) (\n sum without (jvm_memory_pool_name) (jvm_memory_limit_bytes{jvm_memory_type=\"non_heap\", job=~\"$job\",instance=~\"$instance\"} != -1)\n)\nor\navg by (job) (\n sum without (pool) (process_runtime_jvm_memory_limit_bytes{type=\"non_heap\", job=~\"$job\",instance=~\"$instance\"} != -1)\n))) * 100", "format": "time_series", "instant": false, "legendFormat": "{{job}}: JVM memory used(nonheap)", @@ -151,7 +151,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "avg by (job) (\n java_lang_threading_threadcount{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_threads_current{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_threads_live_threads{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n sum without (daemon) (process_runtime_jvm_threads_count{job=~\"$job\",instance=~\"$instance\"})\n)", + "expr": "avg by (job) (\n java_lang_threading_threadcount{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_threads_current{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_threads_live{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n sum without (daemon) (process_runtime_jvm_threads_count{job=~\"$job\",instance=~\"$instance\"})\n)\nor\navg by (job) (\n sum without (jvm_thread_daemon) (jvm_thread_count{job=~\"$job\",instance=~\"$instance\"})\n)", "format": "time_series", "instant": false, "legendFormat": "{{job}}: Threads", @@ -202,7 +202,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "avg by (job) (\n java_lang_classloading_loadedclasscount{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_classes_loaded_classes{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_classes_loaded{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n process_runtime_jvm_classes_loaded_total{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n process_runtime_jvm_classes_loaded{job=~\"$job\",instance=~\"$instance\"}\n)", + "expr": "avg by (job) (\n java_lang_classloading_loadedclasscount{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_class_count{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_classes_loaded_total{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_classes_loaded{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n process_runtime_jvm_classes_loaded_total{job=~\"$job\",instance=~\"$instance\"}\n)", "format": "time_series", "instant": false, "legendFormat": "{{job}}: Classes loaded", @@ -284,7 +284,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "\n java_lang_runtime_uptime{job=~\"$job\",instance=~\"$instance\"}\n/1000\nor\nprocess_uptime_seconds{job=~\"$job\",instance=~\"$instance\"}\nor\nprocess_uptime{job=~\"$job\",instance=~\"$instance\"}\nor\nruntime_uptime_milliseconds_total{job=~\"$job\",instance=~\"$instance\"}\nor\ntime()-process_start_time_seconds{job=~\"$job\",instance=~\"$instance\"}", + "expr": "\n java_lang_runtime_uptime{job=~\"$job\",instance=~\"$instance\"}\n/1000\nor\nprocess_uptime_seconds{job=~\"$job\",instance=~\"$instance\"}\nor\nruntime_uptime_milliseconds_total{job=~\"$job\",instance=~\"$instance\"}\nor\ntime()-process_start_time_seconds{job=~\"$job\",instance=~\"$instance\"}", "format": "time_series", "instant": false, "legendFormat": "{{instance}}: Uptime", @@ -344,7 +344,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "avg by (job,instance) (\n java_lang_runtime_starttime{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job,instance) (\n process_start_time_seconds{job=~\"$job\",instance=~\"$instance\"} * 1000\n)\nor\navg by (job,instance) (\n process_start_time{job=~\"$job\",instance=~\"$instance\"} * 1000\n)", + "expr": "avg by (job,instance) (\n java_lang_runtime_starttime{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job,instance) (\n process_start_time_seconds{job=~\"$job\",instance=~\"$instance\"} * 1000\n)", "format": "time_series", "instant": false, "legendFormat": "{{instance}}: Process start time", @@ -545,7 +545,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "\n avg by (job,instance) (\n rate(java_lang_operatingsystem_processcputime{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])\n)\n/1000000000 * 100\nor\navg by (job,instance) (\n process_cpu_usage{job=~\"$job\",instance=~\"$instance\"} * 100\n)\nor\navg by (job,instance) (\n process_runtime_jvm_cpu_utilization_ratio{job=~\"$job\",instance=~\"$instance\"} * 100\n)\nor\navg by (job,instance) (\n process_runtime_jvm_cpu_utilization{job=~\"$job\",instance=~\"$instance\"} * 100\n)\nor\navg by (job,instance) (\n rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]) * 100\n)", + "expr": "\n avg by (job,instance) (\n rate(java_lang_operatingsystem_processcputime{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])\n)\n/1000000000 * 100\nor\navg by (job,instance) (\n process_cpu_usage{job=~\"$job\",instance=~\"$instance\"} * 100\n)\nor\navg by (job,instance) (\n process_runtime_jvm_cpu_utilization_ratio{job=~\"$job\",instance=~\"$instance\"} * 100\n)\nor\navg by (job,instance) (\n rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]) * 100\n)", "format": "time_series", "instant": false, "legendFormat": "{{instance}}: CPU usage (process)", @@ -556,7 +556,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "\n avg by (job,instance) (\n java_lang_operatingsystem_cpuload{job=~\"$job\",instance=~\"$instance\"}\n)\n* 100\nor\navg by (job,instance) (\n process_runtime_jvm_system_cpu_utilization_ratio{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job,instance) (\n process_runtime_jvm_system_cpu_utilization{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job,instance) (\n system_cpu_usage{job=~\"$job\",instance=~\"$instance\"} * 100\n)", + "expr": "\n avg by (job,instance) (\n java_lang_operatingsystem_cpuload{job=~\"$job\",instance=~\"$instance\"}\n)\n* 100\nor\navg by (job,instance) (\n process_runtime_jvm_system_cpu_utilization_ratio{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job,instance) (\n system_cpu_usage{job=~\"$job\",instance=~\"$instance\"} * 100\n)", "format": "time_series", "instant": false, "legendFormat": "{{instance}}: CPU usage (system)", @@ -870,7 +870,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "avg by (job) (\n java_lang_memory_heapmemoryusage_used{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_bytes_used{area=\"heap\", job=~\"$job\",instance=~\"$instance\"})\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_used_bytes{area=\"heap\", job=~\"$job\",instance=~\"$instance\"})\n)\nor\navg by (job) (\n sum without (pool) (process_runtime_jvm_memory_usage_bytes{type=\"heap\", job=~\"$job\",instance=~\"$instance\"})\n)\nor\navg by (job) (\n sum without (pool) (process_runtime_jvm_memory_usage{type=\"heap\", job=~\"$job\",instance=~\"$instance\"})\n)", + "expr": "avg by (job) (\n java_lang_memory_heapmemoryusage_used{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_used_bytes{area=\"heap\", job=~\"$job\",instance=~\"$instance\"})\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_used{area=\"heap\", job=~\"$job\",instance=~\"$instance\"})\n)\nor\navg by (job) (\n sum without (jvm_memory_pool_name) (jvm_memory_used_bytes{jvm_memory_type=\"heap\", job=~\"$job\",instance=~\"$instance\"})\n)\nor\navg by (job) (\n sum without (pool) (process_runtime_jvm_memory_usage_bytes{type=\"heap\", job=~\"$job\",instance=~\"$instance\"})\n)", "format": "time_series", "instant": false, "legendFormat": "{{job}}: JVM memory used(heap)", @@ -881,7 +881,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "avg by (job) (\n java_lang_memory_heapmemoryusage_max{job=~\"$job\",instance=~\"$instance\"} != -1\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_bytes_max{area=\"heap\", job=~\"$job\",instance=~\"$instance\"} != -1)\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_max_bytes{area=\"heap\", job=~\"$job\",instance=~\"$instance\"} != -1)\n)\nor\navg by (job) (\n sum without (pool) (process_runtime_jvm_memory_limit_bytes{type=\"heap\", job=~\"$job\",instance=~\"$instance\"} != -1)\n)\nor\navg by (job) (\n sum without (pool) (process_runtime_jvm_memory_limit{type=\"heap\", job=~\"$job\",instance=~\"$instance\"} != -1)\n)", + "expr": "avg by (job) (\n java_lang_memory_heapmemoryusage_max{job=~\"$job\",instance=~\"$instance\"} != -1\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_max_bytes{area=\"heap\", job=~\"$job\",instance=~\"$instance\"} != -1)\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_max{area=\"heap\", job=~\"$job\",instance=~\"$instance\"} != -1)\n)\nor\navg by (job) (\n sum without (jvm_memory_pool_name) (jvm_memory_limit_bytes{jvm_memory_type=\"heap\", job=~\"$job\",instance=~\"$instance\"} != -1)\n)\nor\navg by (job) (\n sum without (pool) (process_runtime_jvm_memory_limit_bytes{type=\"heap\", job=~\"$job\",instance=~\"$instance\"} != -1)\n)", "format": "time_series", "instant": false, "legendFormat": "{{job}}: JVM memory max(heap)", @@ -892,7 +892,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "avg by (job) (\n java_lang_memory_heapmemoryusage_committed{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_bytes_committed{area=\"heap\", job=~\"$job\",instance=~\"$instance\"})\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_committed_bytes{area=\"heap\", job=~\"$job\",instance=~\"$instance\"})\n)\nor\navg by (job) (\n sum without (pool) (process_runtime_jvm_memory_committed_bytes{type=\"heap\", job=~\"$job\",instance=~\"$instance\"})\n)\nor\navg by (job) (\n sum without (pool) (process_runtime_jvm_memory_committed{type=\"heap\", job=~\"$job\",instance=~\"$instance\"})\n)", + "expr": "avg by (job) (\n java_lang_memory_heapmemoryusage_committed{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_committed_bytes{area=\"heap\", job=~\"$job\",instance=~\"$instance\"})\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_committed{area=\"heap\", job=~\"$job\",instance=~\"$instance\"})\n)\nor\navg by (job) (\n sum without (jvm_memory_pool_name) (jvm_memory_committed_bytes{jvm_memory_type=\"heap\", job=~\"$job\",instance=~\"$instance\"})\n)\nor\navg by (job) (\n sum without (pool) (process_runtime_jvm_memory_committed_bytes{type=\"heap\", job=~\"$job\",instance=~\"$instance\"})\n)", "format": "time_series", "instant": false, "legendFormat": "{{job}}: JVM memory committed(heap)", @@ -1013,7 +1013,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "avg by (job) (\n java_lang_memory_nonheapmemoryusage_used{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_bytes_used{area=\"nonheap\", job=~\"$job\",instance=~\"$instance\"})\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_used_bytes{area=\"nonheap\", job=~\"$job\",instance=~\"$instance\"})\n)\nor\navg by (job) (\n sum without (pool) (process_runtime_jvm_memory_usage_bytes{type=\"non_heap\", job=~\"$job\",instance=~\"$instance\"})\n)\nor\navg by (job) (\n sum without (pool) (process_runtime_jvm_memory_usage{type=\"non_heap\", job=~\"$job\",instance=~\"$instance\"})\n)", + "expr": "avg by (job) (\n java_lang_memory_nonheapmemoryusage_used{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_used_bytes{area=\"nonheap\", job=~\"$job\",instance=~\"$instance\"})\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_used{area=\"nonheap\", job=~\"$job\",instance=~\"$instance\"})\n)\nor\navg by (job) (\n sum without (jvm_memory_pool_name) (jvm_memory_used_bytes{jvm_memory_type=\"non_heap\", job=~\"$job\",instance=~\"$instance\"})\n)\nor\navg by (job) (\n sum without (pool) (process_runtime_jvm_memory_usage_bytes{type=\"non_heap\", job=~\"$job\",instance=~\"$instance\"})\n)", "format": "time_series", "instant": false, "legendFormat": "{{job}}: JVM memory used(nonheap)", @@ -1024,7 +1024,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "avg by (job) (\n java_lang_memory_nonheapmemoryusage_max{job=~\"$job\",instance=~\"$instance\"} != -1\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_bytes_max{area=\"nonheap\", job=~\"$job\",instance=~\"$instance\"} != -1)\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_max_bytes{area=\"nonheap\", job=~\"$job\",instance=~\"$instance\"} != -1)\n)\nor\navg by (job) (\n sum without (pool) (process_runtime_jvm_memory_limit_bytes{type=\"non_heap\", job=~\"$job\",instance=~\"$instance\"} != -1)\n)\nor\navg by (job) (\n sum without (pool) (process_runtime_jvm_memory_limit{type=\"non_heap\", job=~\"$job\",instance=~\"$instance\"} != -1)\n)", + "expr": "avg by (job) (\n java_lang_memory_nonheapmemoryusage_max{job=~\"$job\",instance=~\"$instance\"} != -1\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_max_bytes{area=\"nonheap\", job=~\"$job\",instance=~\"$instance\"} != -1)\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_max{area=\"nonheap\", job=~\"$job\",instance=~\"$instance\"} != -1)\n)\nor\navg by (job) (\n sum without (jvm_memory_pool_name) (jvm_memory_limit_bytes{jvm_memory_type=\"non_heap\", job=~\"$job\",instance=~\"$instance\"} != -1)\n)\nor\navg by (job) (\n sum without (pool) (process_runtime_jvm_memory_limit_bytes{type=\"non_heap\", job=~\"$job\",instance=~\"$instance\"} != -1)\n)", "format": "time_series", "instant": false, "legendFormat": "{{job}}: JVM memory max(nonheap)", @@ -1035,7 +1035,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "avg by (job) (\n java_lang_memory_nonheapmemoryusage_committed{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_bytes_committed{area=\"nonheap\", job=~\"$job\",instance=~\"$instance\"})\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_committed_bytes{area=\"nonheap\", job=~\"$job\",instance=~\"$instance\"})\n)\nor\navg by (job) (\n sum without (pool) (process_runtime_jvm_memory_committed_bytes{type=\"non_heap\", job=~\"$job\",instance=~\"$instance\"})\n)\nor\navg by (job) (\n sum without (pool) (process_runtime_jvm_memory_committed{type=\"non_heap\", job=~\"$job\",instance=~\"$instance\"})\n)", + "expr": "avg by (job) (\n java_lang_memory_nonheapmemoryusage_committed{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_committed_bytes{area=\"nonheap\", job=~\"$job\",instance=~\"$instance\"})\n)\nor\navg by (job) (\n sum without (id) (jvm_memory_committed{area=\"nonheap\", job=~\"$job\",instance=~\"$instance\"})\n)\nor\navg by (job) (\n sum without (jvm_memory_pool_name) (jvm_memory_committed_bytes{jvm_memory_type=\"non_heap\", job=~\"$job\",instance=~\"$instance\"})\n)\nor\navg by (job) (\n sum without (pool) (process_runtime_jvm_memory_committed_bytes{type=\"non_heap\", job=~\"$job\",instance=~\"$instance\"})\n)", "format": "time_series", "instant": false, "legendFormat": "{{job}}: JVM memory committed(nonheap)", @@ -1167,7 +1167,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "avg by (job) (\n jvm_memory_pool_bytes_used{pool=~\"(G1 |PS )?Eden Space\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_memory_pool_used_bytes{pool=~\"(G1 |PS )?Eden Space\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_memory_used_bytes{id=~\"(G1 |PS )?Eden Space\", area=\"heap\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n process_runtime_jvm_memory_usage_bytes{type=\"heap\", pool=~\"(G1 |PS )?Eden Space\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n process_runtime_jvm_memory_usage{type=\"heap\", pool=~\"(G1 |PS )?Eden Space\", job=~\"$job\",instance=~\"$instance\"}\n)", + "expr": "avg by (job) (\n jvm_memory_pool_used_bytes{pool=~\"(G1 |PS )?Eden Space\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_memory_used_bytes{id=~\"(G1 |PS )?Eden Space\", area=\"heap\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_memory_used_bytes{jvm_memory_type=\"heap\", jvm_memory_pool_name=~\"(G1 |PS )?Eden Space\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n process_runtime_jvm_memory_usage_bytes{type=\"heap\", pool=~\"(G1 |PS )?Eden Space\", job=~\"$job\",instance=~\"$instance\"}\n)", "format": "time_series", "instant": false, "legendFormat": "{{job}}: Eden Space (used)", @@ -1178,7 +1178,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "\n avg by (job) (\n jvm_memory_max_bytes{id=~\"(G1 )?Eden Space\", area=\"heap\", job=~\"$job\",instance=~\"$instance\"}\n)\n != -1\nor\n\n avg by (job) (\n process_runtime_jvm_memory_limit_bytes{pool=~\"(G1 |PS )?Eden Space\", type=\"heap\", job=~\"$job\",instance=~\"$instance\"}\n)\n != -1\nor\n\n avg by (job) (\n process_runtime_jvm_memory_limit{pool=~\"(G1 |PS )?Eden Space\", type=\"heap\", job=~\"$job\",instance=~\"$instance\"}\n)\n != -1\nor\navg by (job) (\n jvm_memory_pool_bytes_max{pool=~\"(G1 |PS )?Eden Space\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_memory_pool_max_bytes{pool=~\"(G1 |PS )?Eden Space\", job=~\"$job\",instance=~\"$instance\"}\n)", + "expr": "\n avg by (job) (\n jvm_memory_limit_bytes{jvm_memory_pool_name=~\"(G1 |PS )?Eden Space\", jvm_memory_type=\"heap\", job=~\"$job\",instance=~\"$instance\"}\n)\n != -1\nor\n\n avg by (job) (\n jvm_memory_max{id=~\"(G1 )?Eden Space\", area=\"heap\", job=~\"$job\",instance=~\"$instance\"}\n)\n != -1\nor\n\n avg by (job) (\n process_runtime_jvm_memory_limit_bytes{pool=~\"(G1 |PS )?Eden Space\", type=\"heap\", job=~\"$job\",instance=~\"$instance\"}\n)\n != -1\nor\navg by (job) (\n jvm_memory_pool_max_bytes{pool=~\"(G1 |PS )?Eden Space\", job=~\"$job\",instance=~\"$instance\"}\n)", "format": "time_series", "instant": false, "legendFormat": "{{job}}: Eden Space (max)", @@ -1189,7 +1189,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "avg by (job) (\n jvm_memory_committed_bytes{id=~\"(G1 |PS )?Eden Space\", area=\"heap\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_memory_pool_bytes_committed{pool=~\"(G1 |PS )?Eden Space\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_memory_pool_committed_bytes{pool=~\"(G1 |PS )?Eden Space\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n process_runtime_jvm_memory_committed_bytes{pool=~\"(G1 |PS )?Eden Space\", type=\"heap\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n process_runtime_jvm_memory_committed{pool=~\"(G1 |PS )?Eden Space\", type=\"heap\", job=~\"$job\",instance=~\"$instance\"}\n)", + "expr": "avg by (job) (\n jvm_memory_committed_bytes{jvm_memory_pool_name=~\"(G1 |PS )?Eden Space\", jvm_memory_type=\"heap\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_memory_committed{id=~\"(G1 |PS )?Eden Space\", area=\"heap\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_memory_pool_committed_bytes{pool=~\"(G1 |PS )?Eden Space\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n process_runtime_jvm_memory_committed_bytes{pool=~\"(G1 |PS )?Eden Space\", type=\"heap\", job=~\"$job\",instance=~\"$instance\"}\n)", "format": "time_series", "instant": false, "legendFormat": "{{job}}: Eden Space (committed)", @@ -1308,7 +1308,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "avg by (job) (\n jvm_memory_pool_bytes_used{pool=~\"(G1 |PS )?Survivor Space\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_memory_pool_used_bytes{pool=~\"(G1 |PS )?Survivor Space\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_memory_used_bytes{id=~\"(G1 |PS )?Survivor Space\", area=\"heap\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n process_runtime_jvm_memory_usage_bytes{type=\"heap\", pool=~\"(G1 |PS )?Survivor Space\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n process_runtime_jvm_memory_usage{type=\"heap\", pool=~\"(G1 |PS )?Survivor Space\", job=~\"$job\",instance=~\"$instance\"}\n)", + "expr": "avg by (job) (\n jvm_memory_pool_used_bytes{pool=~\"(G1 |PS )?Survivor Space\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_memory_used_bytes{jvm_memory_type=\"heap\", jvm_memory_pool_name=~\"(G1 |PS )?Survivor Space\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_memory_used{id=~\"(G1 |PS )?Survivor Space\", area=\"heap\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n process_runtime_jvm_memory_usage_bytes{type=\"heap\", pool=~\"(G1 |PS )?Survivor Space\", job=~\"$job\",instance=~\"$instance\"}\n)", "format": "time_series", "instant": false, "legendFormat": "{{job}}: Survival space (used)", @@ -1319,7 +1319,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "avg by (job) (\n jvm_memory_max_bytes{id=~\"(G1 |PS )?Survivor Space\", area=\"heap\", job=~\"$job\",instance=~\"$instance\"} != -1\n)\nor\navg by (job) (\n jvm_memory_pool_bytes_max{pool=~\"(G1 |PS )?Survivor Space\", job=~\"$job\",instance=~\"$instance\"} != -1\n)\nor\navg by (job) (\n jvm_memory_pool_max_bytes{pool=~\"(G1 |PS )?Survivor Space\", job=~\"$job\",instance=~\"$instance\"} != -1\n)\nor\navg by (job) (\n process_runtime_jvm_memory_limit_bytes{pool=~\"(G1 |PS )?Survivor Space\", type=\"heap\", job=~\"$job\",instance=~\"$instance\"} != -1\n)\nor\navg by (job) (\n process_runtime_jvm_memory_limit{pool=~\"(G1 |PS )?Survivor Space\", type=\"heap\", job=~\"$job\",instance=~\"$instance\"} != -1\n)", + "expr": "avg by (job) (\n jvm_memory_limit_bytes{jvm_memory_pool_name=~\"(G1 |PS )?Survivor Space\", jvm_memory_type=\"heap\", job=~\"$job\",instance=~\"$instance\"} != -1\n)\nor\navg by (job) (\n jvm_memory_max{id=~\"(G1 |PS )?Survivor Space\", area=\"heap\", job=~\"$job\",instance=~\"$instance\"} != -1\n)\nor\navg by (job) (\n jvm_memory_pool_max_bytes{pool=~\"(G1 |PS )?Survivor Space\", job=~\"$job\",instance=~\"$instance\"} != -1\n)\nor\navg by (job) (\n process_runtime_jvm_memory_limit_bytes{pool=~\"(G1 |PS )?Survivor Space\", type=\"heap\", job=~\"$job\",instance=~\"$instance\"} != -1\n)", "format": "time_series", "instant": false, "legendFormat": "{{job}}: Survival space (max)", @@ -1330,7 +1330,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "avg by (job) (\n jvm_memory_committed_bytes{id=~\"(G1 |PS )?Survivor Space\", area=\"heap\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_memory_pool_bytes_committed{pool=~\"(G1 |PS )?Survivor Space\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_memory_pool_committed_bytes{pool=~\"(G1 |PS )?Survivor Space\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n process_runtime_jvm_memory_committed_bytes{pool=~\"(G1 |PS )?Survivor Space\", type=\"heap\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n process_runtime_jvm_memory_committed{pool=~\"(G1 |PS )?Survivor Space\", type=\"heap\", job=~\"$job\",instance=~\"$instance\"}\n)", + "expr": "avg by (job) (\n jvm_memory_committed_bytes{jvm_memory_pool_name=~\"(G1 |PS )?Survivor Space\", jvm_memory_type=\"heap\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_memory_committed{id=~\"(G1 |PS )?Survivor Space\", area=\"heap\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_memory_pool_committed_bytes{pool=~\"(G1 |PS )?Survivor Space\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n process_runtime_jvm_memory_committed_bytes{pool=~\"(G1 |PS )?Survivor Space\", type=\"heap\", job=~\"$job\",instance=~\"$instance\"}\n)", "format": "time_series", "instant": false, "legendFormat": "{{job}}: Survival space (committed)", @@ -1449,7 +1449,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "avg by (job) (\n jvm_memory_pool_bytes_used{pool=\"G1 Old Gen\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_memory_pool_used_bytes{pool=\"PS Old Gen\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_memory_used_bytes{id=\"Tenured Gen\", area=\"heap\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n process_runtime_jvm_memory_usage_bytes{type=\"heap\", pool=~\"G1 Old Gen|Tenured Gen\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n process_runtime_jvm_memory_usage{type=\"heap\", pool=~\"G1 Old Gen|Tenured Gen\", job=~\"$job\",instance=~\"$instance\"}\n)", + "expr": "avg by (job) (\n jvm_memory_pool_used_bytes{pool=\"PS Old Gen\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_memory_used_bytes{jvm_memory_type=\"heap\", jvm_memory_pool_name=~\"G1 Old Gen|Tenured Gen\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_memory_used{id=\"Tenured Gen\", area=\"heap\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n process_runtime_jvm_memory_usage_bytes{type=\"heap\", pool=~\"G1 Old Gen|Tenured Gen\", job=~\"$job\",instance=~\"$instance\"}\n)", "format": "time_series", "instant": false, "legendFormat": "{{job}}: Tenured space (used)", @@ -1460,7 +1460,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "\n avg by (job) (\n jvm_memory_max_bytes{id=\"Tenured Gen\", area=\"heap\", job=~\"$job\",instance=~\"$instance\"}\n)\n != -1\nor\n\n avg by (job) (\n jvm_memory_pool_bytes_max{pool=\"G1 Old Gen\", job=~\"$job\",instance=~\"$instance\"}\n)\n != -1\nor\n\n avg by (job) (\n jvm_memory_pool_used_bytes{pool=\"PS Old Gen\", job=~\"$job\",instance=~\"$instance\"}\n)\n != -1\nor\n\n avg by (job) (\n process_runtime_jvm_memory_limit_bytes{pool=~\"G1 Old Gen|Tenured Gen\", type=\"heap\", job=~\"$job\",instance=~\"$instance\"}\n)\n != -1\nor\n\n avg by (job) (\n process_runtime_jvm_memory_limit{pool=~\"G1 Old Gen|Tenured Gen\", type=\"heap\", job=~\"$job\",instance=~\"$instance\"}\n)\n != -1", + "expr": "\n avg by (job) (\n jvm_memory_limit_bytes{jvm_memory_pool_name=~\"G1 Old Gen|Tenured Gen\", jvm_memory_type=\"heap\", job=~\"$job\",instance=~\"$instance\"}\n)\n != -1\nor\n\n avg by (job) (\n jvm_memory_max{id=\"Tenured Gen\", area=\"heap\", job=~\"$job\",instance=~\"$instance\"}\n)\n != -1\nor\n\n avg by (job) (\n jvm_memory_pool_used_bytes{pool=\"PS Old Gen\", job=~\"$job\",instance=~\"$instance\"}\n)\n != -1\nor\n\n avg by (job) (\n process_runtime_jvm_memory_limit_bytes{pool=~\"G1 Old Gen|Tenured Gen\", type=\"heap\", job=~\"$job\",instance=~\"$instance\"}\n)\n != -1", "format": "time_series", "instant": false, "legendFormat": "{{job}}: Tenured space (max)", @@ -1471,7 +1471,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "avg by (job) (\n jvm_memory_committed_bytes{id=\"Tenured Gen\", area=\"heap\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_memory_pool_bytes_committed{pool=\"G1 Old Gen\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_memory_pool_committed_bytes{pool=\"PS Old Gen\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n process_runtime_jvm_memory_committed_bytes{pool=~\"G1 Old Gen|Tenured Gen\", type=\"heap\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n process_runtime_jvm_memory_committed{pool=~\"G1 Old Gen|Tenured Gen\", type=\"heap\", job=~\"$job\",instance=~\"$instance\"}\n)", + "expr": "avg by (job) (\n jvm_memory_committed_bytes{jvm_memory_pool_name=~\"G1 Old Gen|Tenured Gen\", jvm_memory_type=\"heap\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_memory_committed{id=\"Tenured Gen\", area=\"heap\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_memory_pool_committed_bytes{pool=\"PS Old Gen\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n process_runtime_jvm_memory_committed_bytes{pool=~\"G1 Old Gen|Tenured Gen\", type=\"heap\", job=~\"$job\",instance=~\"$instance\"}\n)", "format": "time_series", "instant": false, "legendFormat": "{{job}}: Tenured space (committed)", @@ -1535,10 +1535,10 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "avg by (job,gc) (\n rate(jvm_gc_collection_seconds_count{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])\n)\nor\nlabel_replace(\n avg by (job,action,cause) (\n rate(jvm_gc_pause_seconds_count{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])\n)\n, \"action\", \"$1\", \"action\", \"end of (.+)\")\nor\nlabel_replace(\n avg by (job,gc,action) (\n rate(process_runtime_jvm_gc_duration_count{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])\n)\n, \"action\", \"$1\", \"action\", \"end of (.+)\")\nor\nlabel_replace(\n avg by (job,gc,action) (\n rate(process_runtime_jvm_gc_duration_seconds_count{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])\n)\n, \"action\", \"$1\", \"action\", \"end of (.+)\")", + "expr": "avg by (job,gc) (\n rate(jvm_gc_collection_seconds_count{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])\n)\nor\navg by (job,jvm_gc_action,jvm_gc_name) (\n rate(jvm_gc_duration_seconds_count{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])\n)\nor\nlabel_replace(\n avg by (job,action,cause) (\n rate(jvm_gc_pause_count{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])\n)\n, \"action\", \"$1\", \"action\", \"end of (.+)\")\nor\nlabel_replace(\n avg by (job,gc,action) (\n rate(process_runtime_jvm_gc_duration_seconds_count{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])\n)\n, \"action\", \"$1\", \"action\", \"end of (.+)\")", "format": "time_series", "instant": false, - "legendFormat": "{{gc}}", + "legendFormat": "{{jvm_gc_name}}", "refId": "Garbage collections" } ], @@ -1623,10 +1623,10 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "avg by (job, gc) (\n rate(jvm_gc_collection_seconds_sum{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])\n /\n rate(jvm_gc_collection_seconds_count{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])\n)\n\nor\nlabel_replace(\n avg by (job, action, cause) (\n rate(jvm_gc_pause_seconds_sum{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])\n /\n rate(jvm_gc_pause_seconds_count{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])\n)\n\n, \"action\", \"$1\", \"action\", \"end of (.+)\")", + "expr": "avg by (job, gc) (\n rate(jvm_gc_collection_seconds_sum{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])\n /\n rate(jvm_gc_collection_seconds_count{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])\n)\n\nor\nlabel_replace(\n avg by (job, action, cause) (\n rate(jvm_gc_pause_sum{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])\n /\n rate(jvm_gc_pause_count{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])\n)\n\n, \"action\", \"$1\", \"action\", \"end of (.+)\")", "format": "time_series", "instant": false, - "legendFormat": "{{ job }}: {{ action }} ({{ cause }}) (avg)", + "legendFormat": "{{job}}: {{ action }} ({{ cause }}) (avg)", "refId": "GC time (avg)" }, { @@ -1634,10 +1634,10 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "label_replace(\n avg by (action,cause, job) (rate(jvm_gc_pause_seconds_max{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))\n, \"action\", \"$1\", \"action\", \"end of (.+)\")", + "expr": "label_replace(\n avg by (action,cause, job) (rate(jvm_gc_pause_max{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))\n, \"action\", \"$1\", \"action\", \"end of (.+)\")", "format": "time_series", "instant": false, - "legendFormat": "{{ job }}: {{ action }} ({{ cause }}) (max)", + "legendFormat": "{{job}}: {{ action }} ({{ cause }}) (max)", "refId": "GC time (max)" }, { @@ -1645,10 +1645,10 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "label_replace(\n histogram_quantile(0.95, sum(rate(process_runtime_jvm_gc_duration_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])) by (le,job,action,gc))\n\n, \"action\", \"$1\", \"action\", \"end of (.+)\")\nor\nlabel_replace(\n histogram_quantile(0.95, sum(rate(process_runtime_jvm_gc_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])) by (le,job,action,gc))\n\n, \"action\", \"$1\", \"action\", \"end of (.+)\")", + "expr": "histogram_quantile(0.95, sum(rate(jvm_gc_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])) by (le,job,jvm_gc_action,jvm_gc_name))\n\nor\nlabel_replace(\n histogram_quantile(0.95, sum(rate(process_runtime_jvm_gc_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])) by (le,job,action,gc))\n\n, \"action\", \"$1\", \"action\", \"end of (.+)\")", "format": "time_series", "instant": false, - "legendFormat": "{{ job }}: {{ action }} ({{ gc }}) (p95)", + "legendFormat": "{{job}}: {{ action }} ({{ gc }}) (p95)", "refId": "GC time (p95)" } ], @@ -1764,7 +1764,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "avg by (job) (\n rate(jvm_gc_memory_allocated_bytes_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])\n)\nor\navg by (job) (\n rate(jvm_memory_pool_allocated_bytes_total{pool=~\"(G1 |PS )?Eden Space\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])\n)", + "expr": "avg by (job) (\n rate(jvm_gc_memory_allocated{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])\n)\nor\navg by (job) (\n rate(jvm_memory_pool_allocated_bytes_total{pool=~\"(G1 |PS )?Eden Space\",job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])\n)", "format": "time_series", "instant": false, "legendFormat": "{{job}}: Allocated (bytes)", @@ -1775,10 +1775,10 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "topk(10,\n avg by (job, arena, thread_name) (rate(process_runtime_jvm_memory_allocation_count{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))\n)\n", + "expr": "topk(10,\n avg by (job, arena, thread_name) (rate(process_runtime_jvm_memory_allocation_bytes_count{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))\n)\n", "format": "time_series", "instant": false, - "legendFormat": "{{ job }}: Allocated ({{ thread_name }}, {{ arena }})", + "legendFormat": "{{job}}: Allocated ({{ thread_name }}, {{ arena }})", "refId": "Allocated" }, { @@ -1786,7 +1786,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "avg by (job) (\n rate(jvm_gc_memory_promoted_bytes_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])\n)", + "expr": "avg by (job) (\n rate(jvm_gc_memory_promoted{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])\n)", "format": "time_series", "instant": false, "legendFormat": "{{job}}: Promoted", @@ -1927,7 +1927,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "avg by (job) (\n java_lang_threading_threadcount{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_threads_current{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_threads_live_threads{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n sum without (daemon) (process_runtime_jvm_threads_count{job=~\"$job\",instance=~\"$instance\"})\n)", + "expr": "avg by (job) (\n java_lang_threading_threadcount{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_threads_current{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_threads_live{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n sum without (daemon) (process_runtime_jvm_threads_count{job=~\"$job\",instance=~\"$instance\"})\n)\nor\navg by (job) (\n sum without (jvm_thread_daemon) (jvm_thread_count{job=~\"$job\",instance=~\"$instance\"})\n)", "format": "time_series", "instant": false, "legendFormat": "{{job}}: Threads", @@ -1938,7 +1938,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "avg by (job) (\n java_lang_threading_daemonthreadcount{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_threads_daemon_threads{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_threads_daemon{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n process_runtime_jvm_threads_count{daemon=\"true\", job=~\"$job\",instance=~\"$instance\"}\n)", + "expr": "avg by (job) (\n java_lang_threading_daemonthreadcount{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_thread_count{jvm_thread_daemon=\"true\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_threads_daemon{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n process_runtime_jvm_threads_count{daemon=\"true\", job=~\"$job\",instance=~\"$instance\"}\n)", "format": "time_series", "instant": false, "legendFormat": "{{job}}: Threads (daemon)", @@ -1949,7 +1949,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "avg by (job) (\n java_lang_threading_peakthreadcount{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_threads_peak_threads{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_threads_peak{job=~\"$job\",instance=~\"$instance\"}\n)", + "expr": "avg by (job) (\n java_lang_threading_peakthreadcount{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_threads_peak{job=~\"$job\",instance=~\"$instance\"}\n)", "format": "time_series", "instant": false, "legendFormat": "{{job}}: Threads (peak)", @@ -2024,7 +2024,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (state, job) (jvm_threads_states_threads{job=~\"$job\",instance=~\"$instance\"})\nor\nsum by (state, job) (jvm_threads_state{job=~\"$job\",instance=~\"$instance\"})", + "expr": "sum by (jvm_thread_state, jvm_thread_daemon, job) (jvm_thread_count{job=~\"$job\",instance=~\"$instance\"})\nor\nsum by (state, job) (jvm_threads_states{job=~\"$job\",instance=~\"$instance\"})\nor\nsum by (state, job) (jvm_threads_state{job=~\"$job\",instance=~\"$instance\"})", "format": "time_series", "instant": false, "legendFormat": "{{ state }}", @@ -2143,7 +2143,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "avg by (job) (\n jvm_buffer_memory_used_bytes{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_buffer_pool_used_bytes{pool=\"direct\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n process_runtime_jvm_buffer_usage_bytes{pool=\"direct\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n process_runtime_jvm_buffer_usage{pool=\"direct\", job=~\"$job\",instance=~\"$instance\"}\n)", + "expr": "avg by (job) (\n jvm_buffer_memory_used_bytes{jvm_buffer_pool_name=\"direct\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_buffer_memory_used{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_buffer_pool_used_bytes{pool=\"direct\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n process_runtime_jvm_buffer_usage_bytes{pool=\"direct\", job=~\"$job\",instance=~\"$instance\"}\n)", "format": "time_series", "instant": false, "legendFormat": "{{job}}: Direct buffer used bytes", @@ -2154,7 +2154,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "avg by (job) (\n jvm_buffer_pool_capacity_bytes{pool=\"direct\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_buffer_total_capacity_bytes{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n process_runtime_jvm_buffer_limit_bytes{pool=\"direct\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n process_runtime_jvm_buffer_limit{pool=\"direct\", job=~\"$job\",instance=~\"$instance\"}\n)", + "expr": "avg by (job) (\n jvm_buffer_memory_limit_bytes{jvm_buffer_pool_name=\"direct\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_buffer_pool_capacity_bytes{pool=\"direct\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_buffer_total_capacity{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n process_runtime_jvm_buffer_limit_bytes{pool=\"direct\", job=~\"$job\",instance=~\"$instance\"}\n)", "format": "time_series", "instant": false, "legendFormat": "{{job}}: Direct buffer capacity", @@ -2261,7 +2261,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "avg by (job) (\n jvm_buffer_memory_used_bytes{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_buffer_pool_used_bytes{pool=\"mapped\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n process_runtime_jvm_buffer_usage_bytes{pool=\"mapped\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n process_runtime_jvm_buffer_usage{pool=\"mapped\", job=~\"$job\",instance=~\"$instance\"}\n)", + "expr": "avg by (job) (\n jvm_buffer_memory_used_bytes{jvm_buffer_pool_name=\"mapped\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_buffer_memory_used{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_buffer_pool_used_bytes{pool=\"mapped\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n process_runtime_jvm_buffer_usage_bytes{pool=\"mapped\", job=~\"$job\",instance=~\"$instance\"}\n)", "format": "time_series", "instant": false, "legendFormat": "{{job}}: Mapped buffer used", @@ -2272,7 +2272,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "avg by (job) (\n jvm_buffer_pool_capacity_bytes{pool=\"mapped\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_buffer_total_capacity_bytes{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n process_runtime_jvm_buffer_limit_bytes{pool=\"mapped\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n process_runtime_jvm_buffer_limit{pool=\"mapped\", job=~\"$job\",instance=~\"$instance\"}\n)", + "expr": "avg by (job) (\n jvm_buffer_memory_limit_bytes{jvm_buffer_pool_name=\"mapped\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_buffer_pool_capacity_bytes{pool=\"mapped\", job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n jvm_buffer_total_capacity{job=~\"$job\",instance=~\"$instance\"}\n)\nor\navg by (job) (\n process_runtime_jvm_buffer_limit_bytes{pool=\"mapped\", job=~\"$job\",instance=~\"$instance\"}\n)", "format": "time_series", "instant": false, "legendFormat": "{{job}}: Mapped buffer capacity", @@ -2307,7 +2307,7 @@ "label": "Job", "multi": true, "name": "job", - "query": "label_values({__name__=~\"jvm_memory_used_bytes|jvm_memory_bytes_max|process_runtime_jvm_memory_usage|process_runtime_jvm_memory_usage_bytes|java_lang_memory_heapmemoryusage_used\"}, job)", + "query": "label_values({__name__=~\"jvm_memory_used|jvm_memory_used_bytes|process_runtime_jvm_memory_usage_bytes|jvm_memory_used_bytes|java_lang_memory_heapmemoryusage_used\"}, job)", "refresh": 2, "sort": 1, "type": "query" @@ -2322,7 +2322,7 @@ "label": "Instance", "multi": true, "name": "instance", - "query": "label_values({__name__=~\"jvm_memory_used_bytes|jvm_memory_bytes_max|process_runtime_jvm_memory_usage|process_runtime_jvm_memory_usage_bytes|java_lang_memory_heapmemoryusage_used\",job=~\"$job\"}, instance)", + "query": "label_values({__name__=~\"jvm_memory_used|jvm_memory_used_bytes|process_runtime_jvm_memory_usage_bytes|jvm_memory_used_bytes|java_lang_memory_heapmemoryusage_used\",job=~\"$job\"}, instance)", "refresh": 2, "sort": 1, "type": "query" diff --git a/jvm-observ-lib/main.libsonnet b/jvm-observ-lib/main.libsonnet index a6e29366f..d3001fcc0 100644 --- a/jvm-observ-lib/main.libsonnet +++ b/jvm-observ-lib/main.libsonnet @@ -15,16 +15,18 @@ local processlib = import 'process-observ-lib/main.libsonnet'; filteringSelector: this.config.filteringSelector, groupLabels: this.config.groupLabels, instanceLabels: this.config.instanceLabels, - uid: this.config.uid, + uid: this.config.uid - '-jvm', dashboardNamePrefix: this.config.dashboardNamePrefix, dashboardTags: this.config.dashboardTags, metricsSource: [] + (if std.member(this.config.metricsSource, 'otel') then ['java_otel'] else []) - + (if std.member(this.config.metricsSource, 'otel_with_suffixes') then ['java_otel_with_suffixes'] else []) + + (if std.member(this.config.metricsSource, 'otel_old') then ['java_otel'] else []) + + (if std.member(this.config.metricsSource, 'otel_old_with_suffixes') then ['java_otel_with_suffixes'] else []) + (if std.member(this.config.metricsSource, 'prometheus') then ['prometheus'] else []) + (if std.member(this.config.metricsSource, 'jmx_exporter') then ['jmx_exporter'] else []) + (if std.member(this.config.metricsSource, 'prometheus_old') then ['prometheus'] else []) + + (if std.member(this.config.metricsSource, 'java_micrometer_with_suffixes') then ['java_micrometer_with_suffixes'] else []) + (if std.member(this.config.metricsSource, 'java_micrometer') then ['java_micrometer'] else []), } ), diff --git a/jvm-observ-lib/prometheus_rules_out/prometheus_alerts.yaml b/jvm-observ-lib/prometheus_rules_out/prometheus_alerts.yaml index 71882cc22..2cf237876 100644 --- a/jvm-observ-lib/prometheus_rules_out/prometheus_alerts.yaml +++ b/jvm-observ-lib/prometheus_rules_out/prometheus_alerts.yaml @@ -8,21 +8,21 @@ groups: expr: |- ((java_lang_memory_heapmemoryusage_used{} or - sum without (id) (jvm_memory_bytes_used{area="heap", }) - or sum without (id) (jvm_memory_used_bytes{area="heap", }) or - sum without (pool) (process_runtime_jvm_memory_usage_bytes{type="heap", }) + sum without (id) (jvm_memory_used{area="heap", }) or - sum without (pool) (process_runtime_jvm_memory_usage{type="heap", }))/(java_lang_memory_heapmemoryusage_max{} != -1 + sum without (jvm_memory_pool_name) (jvm_memory_used_bytes{jvm_memory_type="heap", }) or - sum without (id) (jvm_memory_bytes_max{area="heap", } != -1) + sum without (pool) (process_runtime_jvm_memory_usage_bytes{type="heap", }))/(java_lang_memory_heapmemoryusage_max{} != -1 or sum without (id) (jvm_memory_max_bytes{area="heap", } != -1) or - sum without (pool) (process_runtime_jvm_memory_limit_bytes{type="heap", } != -1) + sum without (id) (jvm_memory_max{area="heap", } != -1) + or + sum without (jvm_memory_pool_name) (jvm_memory_limit_bytes{jvm_memory_type="heap", } != -1) or - sum without (pool) (process_runtime_jvm_memory_limit{type="heap", } != -1))) * 100 > 80 + sum without (pool) (process_runtime_jvm_memory_limit_bytes{type="heap", } != -1))) * 100 > 80 for: 5m keep_firing_for: 5m labels: diff --git a/jvm-observ-lib/signals/buffers.libsonnet b/jvm-observ-lib/signals/buffers.libsonnet index fd78fab45..9d625688d 100644 --- a/jvm-observ-lib/signals/buffers.libsonnet +++ b/jvm-observ-lib/signals/buffers.libsonnet @@ -8,10 +8,12 @@ function(this) aggLevel: 'group', aggFunction: 'avg', discoveryMetric: { - java_micrometer: 'jvm_buffer_memory_used_bytes', // https://github.com/micrometer-metrics/micrometer/blob/main/micrometer-core/src/main/java/io/micrometer/core/instrument/binder/jvm/JvmMemoryMetrics.java + java_micrometer: 'jvm_buffer_memory_used', // can be seen when otel micrometer bridge is used + java_micrometer_with_suffixes: 'jvm_buffer_memory_used_bytes', // https://github.com/micrometer-metrics/micrometer/blob/main/micrometer-core/src/main/java/io/micrometer/core/instrument/binder/jvm/JvmMemoryMetrics.java + otel_old: 'process_runtime_jvm_buffer_usage', + otel_old_with_suffixes: 'process_runtime_jvm_buffer_usage_bytes', + otel_with_suffixes: 'jvm_buffer_memory_used_bytes', prometheus: 'jvm_buffer_pool_used_bytes', // https://prometheus.github.io/client_java/instrumentation/jvm/#jvm-buffer-pool-metrics - otel: 'process_runtime_jvm_buffer_usage', - otel_with_suffixes: 'process_runtime_jvm_buffer_usage_bytes', prometheus_old: 'jvm_buffer_pool_used_bytes', }, signals: { @@ -23,17 +25,23 @@ function(this) optional: true, sources: { java_micrometer: { + expr: 'jvm_buffer_memory_used{%(queriesSelector)s}', + }, + java_micrometer_with_suffixes: { expr: 'jvm_buffer_memory_used_bytes{%(queriesSelector)s}', }, prometheus: { expr: 'jvm_buffer_pool_used_bytes{pool="direct", %(queriesSelector)s}', }, - otel: { + otel_old: { expr: 'process_runtime_jvm_buffer_usage{pool="direct", %(queriesSelector)s}', }, - otel_with_suffixes: { + otel_old_with_suffixes: { expr: 'process_runtime_jvm_buffer_usage_bytes{pool="direct", %(queriesSelector)s}', }, + otel_with_suffixes: { + expr: 'jvm_buffer_memory_used_bytes{jvm_buffer_pool_name="direct", %(queriesSelector)s}', + }, prometheus_old: { expr: 'jvm_buffer_pool_used_bytes{pool="direct", %(queriesSelector)s}', }, @@ -47,17 +55,23 @@ function(this) optional: true, sources: { java_micrometer: { + expr: 'jvm_buffer_total_capacity{%(queriesSelector)s}', + }, + java_micrometer_with_suffixes: { expr: 'jvm_buffer_total_capacity_bytes{%(queriesSelector)s}', }, prometheus: { expr: 'jvm_buffer_pool_capacity_bytes{pool="direct", %(queriesSelector)s}', }, - otel: { + otel_old: { expr: 'process_runtime_jvm_buffer_limit{pool="direct", %(queriesSelector)s}', }, - otel_with_suffixes: { + otel_old_with_suffixes: { expr: 'process_runtime_jvm_buffer_limit_bytes{pool="direct", %(queriesSelector)s}', }, + otel_with_suffixes: { + expr: 'jvm_buffer_memory_limit_bytes{jvm_buffer_pool_name="direct", %(queriesSelector)s}', + }, prometheus_old: { expr: 'jvm_buffer_pool_capacity_bytes{pool="direct", %(queriesSelector)s}', }, @@ -71,17 +85,23 @@ function(this) optional: true, sources: { java_micrometer: { + expr: 'jvm_buffer_memory_used{%(queriesSelector)s}', + }, + java_micrometer_with_suffixes: { expr: 'jvm_buffer_memory_used_bytes{%(queriesSelector)s}', }, prometheus: { expr: 'jvm_buffer_pool_used_bytes{pool="mapped", %(queriesSelector)s}', }, - otel: { + otel_old: { expr: 'process_runtime_jvm_buffer_usage{pool="mapped", %(queriesSelector)s}', }, - otel_with_suffixes: { + otel_old_with_suffixes: { expr: 'process_runtime_jvm_buffer_usage_bytes{pool="mapped", %(queriesSelector)s}', }, + otel_with_suffixes: { + expr: 'jvm_buffer_memory_used_bytes{jvm_buffer_pool_name="mapped", %(queriesSelector)s}', + }, prometheus_old: { expr: 'jvm_buffer_pool_used_bytes{pool="mapped", %(queriesSelector)s}', }, @@ -95,17 +115,23 @@ function(this) optional: true, sources: { java_micrometer: { + expr: 'jvm_buffer_total_capacity{%(queriesSelector)s}', + }, + java_micrometer_with_suffixes: { expr: 'jvm_buffer_total_capacity_bytes{%(queriesSelector)s}', }, prometheus: { expr: 'jvm_buffer_pool_capacity_bytes{pool="mapped", %(queriesSelector)s}', }, - otel: { + otel_old: { expr: 'process_runtime_jvm_buffer_limit{pool="mapped", %(queriesSelector)s}', }, - otel_with_suffixes: { + otel_old_with_suffixes: { expr: 'process_runtime_jvm_buffer_limit_bytes{pool="mapped", %(queriesSelector)s}', }, + otel_with_suffixes: { + expr: 'jvm_buffer_memory_limit_bytes{jvm_buffer_pool_name="mapped", %(queriesSelector)s}', + }, prometheus_old: { expr: 'jvm_buffer_pool_capacity_bytes{pool="mapped", %(queriesSelector)s}', }, diff --git a/jvm-observ-lib/signals/classes.libsonnet b/jvm-observ-lib/signals/classes.libsonnet index ddbc809d2..f9e958fc4 100644 --- a/jvm-observ-lib/signals/classes.libsonnet +++ b/jvm-observ-lib/signals/classes.libsonnet @@ -8,10 +8,12 @@ function(this) aggLevel: 'group', aggFunction: 'avg', discoveryMetric: { - java_micrometer: 'jvm_classes_loaded_classes', // https://github.com/micrometer-metrics/micrometer/blob/main/micrometer-core/src/main/java/io/micrometer/core/instrument/binder/jvm/ClassLoaderMetrics.java - prometheus: 'jvm_classes_loaded', // https://prometheus.github.io/client_java/instrumentation/jvm/#jvm-class-loading-metrics - otel: 'process_runtime_jvm_classes_loaded', - otel_with_suffixes: 'process_runtime_jvm_classes_loaded_total', + java_micrometer: 'jvm_classes_loaded', // https://github.com/micrometer-metrics/micrometer/blob/main/micrometer-core/src/main/java/io/micrometer/core/instrument/binder/jvm/ClassLoaderMetrics.java + java_micrometer_with_suffixes: 'jvm_classes_loaded_classes', + prometheus: 'jvm_classes_loaded_total', // https://prometheus.github.io/client_java/instrumentation/jvm/#jvm-class-loading-metrics + otel_old: 'process_runtime_jvm_classes_loaded', + otel_old_with_suffixes: 'process_runtime_jvm_classes_loaded_total', + otel_with_suffixes: 'jvm_class_count', prometheus_old: 'jvm_classes_loaded', jmx_exporter: 'java_lang_classloading_loadedclasscount', }, @@ -23,17 +25,23 @@ function(this) unit: 'short', sources: { java_micrometer: { + expr: 'jvm_classes_loaded{%(queriesSelector)s}', + }, + java_micrometer_with_suffixes: { expr: 'jvm_classes_loaded_classes{%(queriesSelector)s}', }, prometheus: { - expr: 'jvm_classes_loaded{%(queriesSelector)s}', + expr: 'jvm_classes_loaded_total{%(queriesSelector)s}', }, - otel: { + otel_old: { expr: 'process_runtime_jvm_classes_loaded{%(queriesSelector)s}', }, - otel_with_suffixes: { + otel_old_with_suffixes: { expr: 'process_runtime_jvm_classes_loaded_total{%(queriesSelector)s}', }, + otel_with_suffixes: { + expr: 'jvm_class_count{%(queriesSelector)s}', + }, prometheus_old: { expr: 'jvm_classes_loaded{%(queriesSelector)s}', }, diff --git a/jvm-observ-lib/signals/gc.libsonnet b/jvm-observ-lib/signals/gc.libsonnet index 23ba5a0a1..7c8f0e670 100644 --- a/jvm-observ-lib/signals/gc.libsonnet +++ b/jvm-observ-lib/signals/gc.libsonnet @@ -6,11 +6,13 @@ function(this) aggLevel: 'group', aggFunction: 'avg', discoveryMetric: { - java_micrometer: 'jvm_memory_used_bytes', + java_micrometer: 'jvm_memory_used', + java_micrometer_with_suffixes: 'jvm_memory_used_bytes', prometheus: 'jvm_memory_used_bytes', // https://prometheus.github.io/client_java/instrumentation/jvm/#jvm-memory-metrics prometheus_old: 'jvm_memory_bytes_used', - otel: 'process_runtime_jvm_memory_usage', //https://opentelemetry.io/docs/specs/semconv/runtime/jvm-metrics/ - otel_with_suffixes: 'process_runtime_jvm_memory_usage_bytes', + otel_old: 'process_runtime_jvm_memory_usage', //https://opentelemetry.io/docs/specs/semconv/runtime/jvm-metrics/ + otel_old_with_suffixes: 'process_runtime_jvm_memory_usage_bytes', + otel_with_suffixes: 'jvm_memory_used_bytes', }, signals: { @@ -28,6 +30,14 @@ function(this) optional: true, sources: { java_micrometer: { + expr: 'jvm_gc_pause_count{%(queriesSelector)s}', + aggKeepLabels: ['action', 'cause'], + legendCustomTemplate: '%(aggLegend)s', + exprWrappers: [ + actionLabelPrettify, + ], + }, + java_micrometer_with_suffixes: { expr: 'jvm_gc_pause_seconds_count{%(queriesSelector)s}', aggKeepLabels: ['action', 'cause'], legendCustomTemplate: '%(aggLegend)s', @@ -41,7 +51,7 @@ function(this) legendCustomTemplate: '%(aggLegend)s', }, prometheus_old: self.prometheus, - otel: { + otel_old: { expr: 'process_runtime_jvm_gc_duration_count{%(queriesSelector)s}', legendCustomTemplate: '%(aggLegend)s', aggKeepLabels: ['gc', 'action'], @@ -49,9 +59,14 @@ function(this) actionLabelPrettify, ], }, - otel_with_suffixes: self.otel { + otel_old_with_suffixes: self.otel_old { expr: 'process_runtime_jvm_gc_duration_seconds_count{%(queriesSelector)s}', }, + otel_with_suffixes: { + expr: 'jvm_gc_duration_seconds_count{%(queriesSelector)s}', + legendCustomTemplate: '%(aggLegend)s', + aggKeepLabels: ['jvm_gc_action', 'jvm_gc_name'], + }, }, }, //gc @@ -63,8 +78,15 @@ function(this) optional: true, sources: { java_micrometer: { + expr: 'avg by (action,cause, %(agg)s) (rate(jvm_gc_pause_max{%(queriesSelector)s}[$__rate_interval]))', + legendCustomTemplate: '%(aggLegend)s: {{ action }} ({{ cause }}) (max)', + exprWrappers: [ + actionLabelPrettify, + ], + }, + java_micrometer_with_suffixes: { expr: 'avg by (action,cause, %(agg)s) (rate(jvm_gc_pause_seconds_max{%(queriesSelector)s}[$__rate_interval]))', - legendCustomTemplate: '{{ %(agg)s }}: {{ action }} ({{ cause }}) (max)', + legendCustomTemplate: '%(aggLegend)s: {{ action }} ({{ cause }}) (max)', exprWrappers: [ actionLabelPrettify, ], @@ -79,6 +101,19 @@ function(this) optional: true, sources: { java_micrometer: { + expr: ||| + avg by (%(agg)s, action, cause) ( + rate(jvm_gc_pause_sum{%(queriesSelector)s}[$__rate_interval]) + / + rate(jvm_gc_pause_count{%(queriesSelector)s}[$__rate_interval]) + ) + |||, + legendCustomTemplate: '%(aggLegend)s: {{ action }} ({{ cause }}) (avg)', + exprWrappers: [ + actionLabelPrettify, + ], + }, + java_micrometer_with_suffixes: { expr: ||| avg by (%(agg)s, action, cause) ( rate(jvm_gc_pause_seconds_sum{%(queriesSelector)s}[$__rate_interval]) @@ -86,7 +121,7 @@ function(this) rate(jvm_gc_pause_seconds_count{%(queriesSelector)s}[$__rate_interval]) ) |||, - legendCustomTemplate: '{{ %(agg)s }}: {{ action }} ({{ cause }}) (avg)', + legendCustomTemplate: '%(aggLegend)s: {{ action }} ({{ cause }}) (avg)', exprWrappers: [ actionLabelPrettify, ], @@ -99,7 +134,7 @@ function(this) rate(jvm_gc_collection_seconds_count{%(queriesSelector)s}[$__rate_interval]) ) |||, - legendCustomTemplate: '{{ %(agg)s }}: {{ gc }} (avg)', + legendCustomTemplate: '%(aggLegend)s: {{ gc }} (avg)', }, prometheus_old: self.prometheus, @@ -112,20 +147,26 @@ function(this) unit: 'ms', optional: true, sources: { - otel: { + otel_old: { expr: ||| histogram_quantile(0.95, sum(rate(process_runtime_jvm_gc_duration_bucket{%(queriesSelector)s}[$__rate_interval])) by (le,%(agg)s,action,gc)) |||, - legendCustomTemplate: '{{ %(agg)s }}: {{ action }} ({{ gc }}) (p95)', + legendCustomTemplate: '%(aggLegend)s: {{ action }} ({{ gc }}) (p95)', exprWrappers: [ actionLabelPrettify, ], }, - otel_with_suffixes: self.otel { + otel_old_with_suffixes: self.otel_old { expr: ||| histogram_quantile(0.95, sum(rate(process_runtime_jvm_gc_duration_seconds_bucket{%(queriesSelector)s}[$__rate_interval])) by (le,%(agg)s,action,gc)) |||, }, + otel_with_suffixes: { + expr: ||| + histogram_quantile(0.95, sum(rate(jvm_gc_duration_seconds_bucket{%(queriesSelector)s}[$__rate_interval])) by (le,%(agg)s,jvm_gc_action,jvm_gc_name)) + |||, + legendCustomTemplate: '%(aggLegend)s: {{ jvm_gc_action }} ({{ jvm_gc_name }}) (p95)', + }, }, }, @@ -138,6 +179,9 @@ function(this) optional: true, sources: { java_micrometer: { + expr: 'jvm_gc_memory_promoted{%(queriesSelector)s}', + }, + java_micrometer_with_suffixes: { expr: 'jvm_gc_memory_promoted_bytes_total{%(queriesSelector)s}', }, }, @@ -151,6 +195,9 @@ function(this) optional: true, sources: { java_micrometer: { + expr: 'jvm_gc_memory_allocated{%(queriesSelector)s}', + }, + java_micrometer_with_suffixes: { expr: 'jvm_gc_memory_allocated_bytes_total{%(queriesSelector)s}', }, prometheus: { @@ -167,15 +214,22 @@ function(this) unit: 'allocs', optional: true, sources: { - otel: { + otel_old: { expr: ||| topk(10, avg by (%(agg)s, arena, thread_name) (rate(process_runtime_jvm_memory_allocation_count{%(queriesSelector)s}[$__rate_interval])) ) |||, - legendCustomTemplate: '{{ %(agg)s }}: Allocated ({{ thread_name }}, {{ arena }})', + legendCustomTemplate: '%(aggLegend)s: Allocated ({{ thread_name }}, {{ arena }})', + }, + otel_with_suffixes: { + expr: ||| + topk(10, + avg by (%(agg)s, arena, thread_name) (rate(process_runtime_jvm_memory_allocation_bytes_count{%(queriesSelector)s}[$__rate_interval])) + ) + |||, + legendCustomTemplate: '%(aggLegend)s: Allocated ({{ thread_name }}, {{ arena }})', }, - otel_with_suffixes: self.otel, }, }, @@ -192,18 +246,24 @@ function(this) java_micrometer: { expr: 'jvm_memory_used_bytes{id=~"(G1 |PS )?Eden Space", area="heap", %(queriesSelector)s}', }, + java_micrometer_with_suffixes: { + expr: 'jvm_memory_used{id=~"(G1 |PS )?Eden Space", area="heap", %(queriesSelector)s}', + }, prometheus: { expr: 'jvm_memory_pool_used_bytes{pool=~"(G1 |PS )?Eden Space", %(queriesSelector)s}', }, prometheus_old: { expr: 'jvm_memory_pool_bytes_used{pool=~"(G1 |PS )?Eden Space", %(queriesSelector)s}', }, - otel: { + otel_old: { expr: 'process_runtime_jvm_memory_usage{type="heap", pool=~"(G1 |PS )?Eden Space", %(queriesSelector)s}', }, - otel_with_suffixes: { + otel_old_with_suffixes: { expr: 'process_runtime_jvm_memory_usage_bytes{type="heap", pool=~"(G1 |PS )?Eden Space", %(queriesSelector)s}', }, + otel_with_suffixes: { + expr: 'jvm_memory_used_bytes{jvm_memory_type="heap", jvm_memory_pool_name=~"(G1 |PS )?Eden Space", %(queriesSelector)s}', + }, }, }, memoryMaxEden: { @@ -223,6 +283,12 @@ function(this) optional: true, sources: { java_micrometer: { + expr: 'jvm_memory_max{id=~"(G1 )?Eden Space", area="heap", %(queriesSelector)s}', + exprWrappers: [ + ['', ' != -1'], + ], + }, + java_micrometer_with_suffixes: { expr: 'jvm_memory_max_bytes{id=~"(G1 )?Eden Space", area="heap", %(queriesSelector)s}', exprWrappers: [ ['', ' != -1'], @@ -234,15 +300,21 @@ function(this) prometheus_old: { expr: 'jvm_memory_pool_bytes_max{pool=~"(G1 |PS )?Eden Space", %(queriesSelector)s}', }, - otel: { + otel_old: { expr: 'process_runtime_jvm_memory_limit{pool=~"(G1 |PS )?Eden Space", type="heap", %(queriesSelector)s}', exprWrappers: [ ['', ' != -1'], ], }, - otel_with_suffixes: self.otel { + otel_old_with_suffixes: self.otel_old { expr: 'process_runtime_jvm_memory_limit_bytes{pool=~"(G1 |PS )?Eden Space", type="heap", %(queriesSelector)s}', }, + otel_with_suffixes: { + expr: 'jvm_memory_limit_bytes{jvm_memory_pool_name=~"(G1 |PS )?Eden Space", jvm_memory_type="heap", %(queriesSelector)s}', + exprWrappers: [ + ['', ' != -1'], + ], + }, }, }, memoryCommittedEden: { @@ -255,6 +327,9 @@ function(this) optional: true, sources: { java_micrometer: { + expr: 'jvm_memory_committed{id=~"(G1 |PS )?Eden Space", area="heap", %(queriesSelector)s}', + }, + java_micrometer_with_suffixes: { expr: 'jvm_memory_committed_bytes{id=~"(G1 |PS )?Eden Space", area="heap", %(queriesSelector)s}', }, prometheus: { @@ -263,12 +338,15 @@ function(this) prometheus_old: { expr: 'jvm_memory_pool_bytes_committed{pool=~"(G1 |PS )?Eden Space", %(queriesSelector)s}', }, - otel: { + otel_old: { expr: 'process_runtime_jvm_memory_committed{pool=~"(G1 |PS )?Eden Space", type="heap", %(queriesSelector)s}', }, - otel_with_suffixes: { + otel_old_with_suffixes: { expr: 'process_runtime_jvm_memory_committed_bytes{pool=~"(G1 |PS )?Eden Space", type="heap", %(queriesSelector)s}', }, + otel_with_suffixes: { + expr: 'jvm_memory_committed_bytes{jvm_memory_pool_name=~"(G1 |PS )?Eden Space", jvm_memory_type="heap", %(queriesSelector)s}', + }, }, }, @@ -283,6 +361,9 @@ function(this) sources: { //spring java_micrometer: { + expr: 'jvm_memory_used{id=~"(G1 |PS )?Survivor Space", area="heap", %(queriesSelector)s}', + }, + java_micrometer_with_suffixes: { expr: 'jvm_memory_used_bytes{id=~"(G1 |PS )?Survivor Space", area="heap", %(queriesSelector)s}', }, prometheus: { @@ -291,12 +372,15 @@ function(this) prometheus_old: { expr: 'jvm_memory_pool_bytes_used{pool=~"(G1 |PS )?Survivor Space", %(queriesSelector)s}', }, - otel: { + otel_old: { expr: 'process_runtime_jvm_memory_usage{type="heap", pool=~"(G1 |PS )?Survivor Space", %(queriesSelector)s}', }, - otel_with_suffixes: { + otel_old_with_suffixes: { expr: 'process_runtime_jvm_memory_usage_bytes{type="heap", pool=~"(G1 |PS )?Survivor Space", %(queriesSelector)s}', }, + otel_with_suffixes: { + expr: 'jvm_memory_used_bytes{jvm_memory_type="heap", jvm_memory_pool_name=~"(G1 |PS )?Survivor Space", %(queriesSelector)s}', + }, }, }, memoryMaxSurvival: { @@ -316,6 +400,9 @@ function(this) optional: true, sources: { java_micrometer: { + expr: 'jvm_memory_max{id=~"(G1 |PS )?Survivor Space", area="heap", %(queriesSelector)s} != -1', + }, + java_micrometer_with_suffixes: { expr: 'jvm_memory_max_bytes{id=~"(G1 |PS )?Survivor Space", area="heap", %(queriesSelector)s} != -1', }, prometheus: { @@ -324,12 +411,15 @@ function(this) prometheus_old: { expr: 'jvm_memory_pool_bytes_max{pool=~"(G1 |PS )?Survivor Space", %(queriesSelector)s} != -1', }, - otel: { + otel_old: { expr: 'process_runtime_jvm_memory_limit{pool=~"(G1 |PS )?Survivor Space", type="heap", %(queriesSelector)s} != -1', }, - otel_with_suffixes: { + otel_old_with_suffixes: { expr: 'process_runtime_jvm_memory_limit_bytes{pool=~"(G1 |PS )?Survivor Space", type="heap", %(queriesSelector)s} != -1', }, + otel_with_suffixes: { + expr: 'jvm_memory_limit_bytes{jvm_memory_pool_name=~"(G1 |PS )?Survivor Space", jvm_memory_type="heap", %(queriesSelector)s} != -1', + }, }, }, memoryCommittedSurvival: { @@ -342,6 +432,9 @@ function(this) optional: true, sources: { java_micrometer: { + expr: 'jvm_memory_committed{id=~"(G1 |PS )?Survivor Space", area="heap", %(queriesSelector)s}', + }, + java_micrometer_with_suffixes: { expr: 'jvm_memory_committed_bytes{id=~"(G1 |PS )?Survivor Space", area="heap", %(queriesSelector)s}', }, prometheus: { @@ -350,12 +443,15 @@ function(this) prometheus_old: { expr: 'jvm_memory_pool_bytes_committed{pool=~"(G1 |PS )?Survivor Space", %(queriesSelector)s}', }, - otel: { + otel_old: { expr: 'process_runtime_jvm_memory_committed{pool=~"(G1 |PS )?Survivor Space", type="heap", %(queriesSelector)s}', }, - otel_with_suffixes: { + otel_old_with_suffixes: { expr: 'process_runtime_jvm_memory_committed_bytes{pool=~"(G1 |PS )?Survivor Space", type="heap", %(queriesSelector)s}', }, + otel_with_suffixes: { + expr: 'jvm_memory_committed_bytes{jvm_memory_pool_name=~"(G1 |PS )?Survivor Space", jvm_memory_type="heap", %(queriesSelector)s}', + }, }, }, @@ -369,6 +465,9 @@ function(this) sources: { //spring java_micrometer: { + expr: 'jvm_memory_used{id="Tenured Gen", area="heap", %(queriesSelector)s}', + }, + java_micrometer_with_suffixes: { expr: 'jvm_memory_used_bytes{id="Tenured Gen", area="heap", %(queriesSelector)s}', }, prometheus: { @@ -377,12 +476,15 @@ function(this) prometheus_old: { expr: 'jvm_memory_pool_bytes_used{pool="G1 Old Gen", %(queriesSelector)s}', }, - otel: { + otel_old: { expr: 'process_runtime_jvm_memory_usage{type="heap", pool=~"G1 Old Gen|Tenured Gen", %(queriesSelector)s}', }, - otel_with_suffixes: { + otel_old_with_suffixes: { expr: 'process_runtime_jvm_memory_usage_bytes{type="heap", pool=~"G1 Old Gen|Tenured Gen", %(queriesSelector)s}', }, + otel_with_suffixes: { + expr: 'jvm_memory_used_bytes{jvm_memory_type="heap", jvm_memory_pool_name=~"G1 Old Gen|Tenured Gen", %(queriesSelector)s}', + }, }, }, memoryMaxTenured: { @@ -402,6 +504,12 @@ function(this) optional: true, sources: { java_micrometer: { + expr: 'jvm_memory_max{id="Tenured Gen", area="heap", %(queriesSelector)s}', + exprWrappers: [ + ['', ' != -1'], + ], + }, + java_micrometer_with_suffixes: { expr: 'jvm_memory_max_bytes{id="Tenured Gen", area="heap", %(queriesSelector)s}', exprWrappers: [ ['', ' != -1'], @@ -419,15 +527,21 @@ function(this) ['', ' != -1'], ], }, - otel: { + otel_old: { expr: 'process_runtime_jvm_memory_limit{pool=~"G1 Old Gen|Tenured Gen", type="heap", %(queriesSelector)s}', exprWrappers: [ ['', ' != -1'], ], }, - otel_with_suffixes: self.otel { + otel_old_with_suffixes: self.otel_old { expr: 'process_runtime_jvm_memory_limit_bytes{pool=~"G1 Old Gen|Tenured Gen", type="heap", %(queriesSelector)s}', }, + otel_with_suffixes: { + expr: 'jvm_memory_limit_bytes{jvm_memory_pool_name=~"G1 Old Gen|Tenured Gen", jvm_memory_type="heap", %(queriesSelector)s}', + exprWrappers: [ + ['', ' != -1'], + ], + }, }, }, memoryCommittedTenured: { @@ -440,6 +554,9 @@ function(this) optional: true, sources: { java_micrometer: { + expr: 'jvm_memory_committed{id="Tenured Gen", area="heap", %(queriesSelector)s}', + }, + java_micrometer_with_suffixes: { expr: 'jvm_memory_committed_bytes{id="Tenured Gen", area="heap", %(queriesSelector)s}', }, prometheus: { @@ -448,12 +565,15 @@ function(this) prometheus_old: { expr: 'jvm_memory_pool_bytes_committed{pool="G1 Old Gen", %(queriesSelector)s}', }, - otel: { + otel_old: { expr: 'process_runtime_jvm_memory_committed{pool=~"G1 Old Gen|Tenured Gen", type="heap", %(queriesSelector)s}', }, - otel_with_suffixes: { + otel_old_with_suffixes: { expr: 'process_runtime_jvm_memory_committed_bytes{pool=~"G1 Old Gen|Tenured Gen", type="heap", %(queriesSelector)s}', }, + otel_with_suffixes: { + expr: 'jvm_memory_committed_bytes{jvm_memory_pool_name=~"G1 Old Gen|Tenured Gen", jvm_memory_type="heap", %(queriesSelector)s}', + }, }, }, }, diff --git a/jvm-observ-lib/signals/hikari.libsonnet b/jvm-observ-lib/signals/hikari.libsonnet index d0ec495a7..8254065ff 100644 --- a/jvm-observ-lib/signals/hikari.libsonnet +++ b/jvm-observ-lib/signals/hikari.libsonnet @@ -9,8 +9,8 @@ function(this) aggFunction: 'avg', discoveryMetric: { java_micrometer: 'hikaricp_connections', // https://github.com/brettwooldridge/HikariCP/blob/dev/src/main/java/com/zaxxer/hikari/metrics/micrometer/MicrometerMetricsTracker.java - otel: 'hikaricp_connections', - otel_with_suffixes: self.otel, + otel_old: 'hikaricp_connections', + otel_old_with_suffixes: self.otel_old, }, signals: { connections: { @@ -24,11 +24,12 @@ function(this) expr: 'hikaricp_connections{%(queriesSelector)s}', aggKeepLabels: ['pool'], }, - otel: { + java_micrometer_with_suffixes: self.java_micrometer, + otel_old: { expr: 'hikaricp_connections{%(queriesSelector)s}', aggKeepLabels: ['pool'], }, - otel_with_suffixes: self.otel, + otel_old_with_suffixes: self.otel_old, }, }, timeouts: { @@ -42,6 +43,10 @@ function(this) expr: 'hikaricp_connections_timeout_total{%(queriesSelector)s}', aggKeepLabels: ['pool'], }, + java_micrometer_with_suffixes: { + expr: 'hikaricp_connections_timeout{%(queriesSelector)s}', + aggKeepLabels: ['pool'], + }, }, }, connectionsActive: { @@ -55,11 +60,12 @@ function(this) expr: 'hikaricp_connections_active{%(queriesSelector)s}', aggKeepLabels: ['pool'], }, - otel: { + java_micrometer_with_suffixes: self.java_micrometer, + otel_old: { expr: 'hikaricp_connections_active{%(queriesSelector)s}', aggKeepLabels: ['pool'], }, - otel_with_suffixes: self.otel, + otel_old_with_suffixes: self.otel_old, }, }, connectionsIdle: { @@ -73,11 +79,12 @@ function(this) expr: 'hikaricp_connections_idle{%(queriesSelector)s}', aggKeepLabels: ['pool'], }, - otel: { + java_micrometer_with_suffixes: self.java_micrometer, + otel_old: { expr: 'hikaricp_connections_idle{%(queriesSelector)s}', aggKeepLabels: ['pool'], }, - otel_with_suffixes: self.otel, + otel_old_with_suffixes: self.otel_old, }, }, connectionsPending: { @@ -91,12 +98,13 @@ function(this) expr: 'hikaricp_connections_pending{%(queriesSelector)s}', aggKeepLabels: ['pool'], }, - otel: { + java_micrometer_with_suffixes: self.java_micrometer, + otel_old: { expr: 'hikaricp_connections_pending{%(queriesSelector)s}', aggKeepLabels: ['pool'], }, - otel_with_suffixes: self.otel, + otel_old_with_suffixes: self.otel_old, }, }, connectionsCreationDurationAvg: { @@ -107,12 +115,20 @@ function(this) optional: true, sources: { java_micrometer: { + expr: ||| + rate(hikaricp_connections_creation_sum{%(queriesSelector)s}[$__rate_interval]) + /rate(hikaricp_connections_creation_count{%(queriesSelector)s}[$__rate_interval]) + |||, + aggKeepLabels: ['pool'], + }, + java_micrometer_with_suffixes: { expr: ||| rate(hikaricp_connections_creation_seconds_sum{%(queriesSelector)s}[$__rate_interval]) /rate(hikaricp_connections_creation_seconds_count{%(queriesSelector)s}[$__rate_interval]) |||, aggKeepLabels: ['pool'], }, + }, }, connectionsUsageDurationAvg: { @@ -123,6 +139,13 @@ function(this) optional: true, sources: { java_micrometer: { + expr: ||| + rate(hikaricp_connections_usage_sum{%(queriesSelector)s}[$__rate_interval]) + /rate(hikaricp_connections_usage_count{%(queriesSelector)s}[$__rate_interval]) + |||, + aggKeepLabels: ['pool'], + }, + java_micrometer_with_suffixes: { expr: ||| rate(hikaricp_connections_usage_seconds_sum{%(queriesSelector)s}[$__rate_interval]) /rate(hikaricp_connections_usage_seconds_count{%(queriesSelector)s}[$__rate_interval]) @@ -139,6 +162,13 @@ function(this) optional: true, sources: { java_micrometer: { + expr: ||| + rate(hikaricp_connections_acquire_sum{%(queriesSelector)s}[$__rate_interval]) + /rate(hikaricp_connections_acquire_count{%(queriesSelector)s}[$__rate_interval]) + |||, + aggKeepLabels: ['pool'], + }, + java_micrometer_with_suffixes: { expr: ||| rate(hikaricp_connections_acquire_seconds_sum{%(queriesSelector)s}[$__rate_interval]) /rate(hikaricp_connections_acquire_seconds_count{%(queriesSelector)s}[$__rate_interval]) @@ -154,7 +184,7 @@ function(this) unit: 's', optional: true, sources: { - otel: { + otel_old: { expr: 'hikaricp_connections_creation_bucket{%(queriesSelector)s}', aggKeepLabels: ['pool'], }, @@ -171,7 +201,7 @@ function(this) unit: 's', optional: true, sources: { - otel: { + otel_old: { expr: 'hikaricp_connections_usage_bucket{%(queriesSelector)s}', aggKeepLabels: ['pool'], }, @@ -188,7 +218,7 @@ function(this) unit: 's', optional: true, sources: { - otel: { + otel_old: { expr: 'hikaricp_connections_acquire_bucket{%(queriesSelector)s}', aggKeepLabels: ['pool'], }, diff --git a/jvm-observ-lib/signals/logback.libsonnet b/jvm-observ-lib/signals/logback.libsonnet index 30f1333aa..63ff23a4e 100644 --- a/jvm-observ-lib/signals/logback.libsonnet +++ b/jvm-observ-lib/signals/logback.libsonnet @@ -20,6 +20,14 @@ function(this) optional: true, sources: { java_micrometer: { + expr: ||| + logback_events{%(queriesSelector)s} + |||, + exprWrappers: [ + ['topk(10,', ')'], + ], + }, + java_micrometer_with_suffixes: { expr: ||| logback_events_total{%(queriesSelector)s} |||, @@ -37,6 +45,12 @@ function(this) optional: true, sources: { java_micrometer: { + expr: 'logback_events{level="error", %(queriesSelector)s}', + exprWrappers: [ + ['topk(10,', ')'], + ], + }, + java_micrometer_with_suffixes: { expr: 'logback_events_total{level="error", %(queriesSelector)s}', exprWrappers: [ ['topk(10,', ')'], diff --git a/jvm-observ-lib/signals/memory.libsonnet b/jvm-observ-lib/signals/memory.libsonnet index 1faf44545..325dc1d11 100644 --- a/jvm-observ-lib/signals/memory.libsonnet +++ b/jvm-observ-lib/signals/memory.libsonnet @@ -6,10 +6,12 @@ function(this) aggLevel: 'group', aggFunction: 'avg', discoveryMetric: { - java_micrometer: 'jvm_memory_used_bytes', + java_micrometer: 'jvm_memory_used', + java_micrometer_with_suffixes: 'jvm_memory_used_bytes', prometheus: 'jvm_memory_used_bytes', // https://prometheus.github.io/client_java/instrumentation/jvm/#jvm-memory-metrics - otel: 'process_runtime_jvm_memory_usage', - otel_with_suffixes: 'process_runtime_jvm_memory_usage_bytes', + otel_old: 'process_runtime_jvm_memory_usage', + otel_old_with_suffixes: 'process_runtime_jvm_memory_usage_bytes', + otel_with_suffixes: 'jvm_memory_used_bytes', prometheus_old: 'jvm_memory_bytes_max', jmx_exporter: 'java_lang_memory_heapmemoryusage_used', //https://github.com/prometheus/jmx_exporter/blob/main/collector/src/test/java/io/prometheus/jmx/JmxCollectorTest.java#L195 }, @@ -23,17 +25,23 @@ function(this) sources: { //spring java_micrometer: { + expr: 'sum without (id) (jvm_memory_used{area="heap", %(queriesSelector)s})', + }, + java_micrometer_with_suffixes: { expr: 'sum without (id) (jvm_memory_used_bytes{area="heap", %(queriesSelector)s})', }, prometheus: { expr: 'sum without (id) (jvm_memory_used_bytes{area="heap", %(queriesSelector)s})', }, - otel: { + otel_old: { expr: 'sum without (pool) (process_runtime_jvm_memory_usage{type="heap", %(queriesSelector)s})', }, - otel_with_suffixes: { + otel_old_with_suffixes: { expr: 'sum without (pool) (process_runtime_jvm_memory_usage_bytes{type="heap", %(queriesSelector)s})', }, + otel_with_suffixes: { + expr: 'sum without (jvm_memory_pool_name) (jvm_memory_used_bytes{jvm_memory_type="heap", %(queriesSelector)s})', + }, prometheus_old: { expr: 'sum without (id) (jvm_memory_bytes_used{area="heap", %(queriesSelector)s})', }, @@ -58,15 +66,21 @@ function(this) unit: 'bytes', sources: { java_micrometer: { + expr: 'sum without (id) (jvm_memory_max{area="heap", %(queriesSelector)s} != -1)', + }, + java_micrometer_with_suffixes: { expr: 'sum without (id) (jvm_memory_max_bytes{area="heap", %(queriesSelector)s} != -1)', }, - prometheus: self.java_micrometer, - otel: { + prometheus: self.java_micrometer_with_suffixes, + otel_old: { expr: 'sum without (pool) (process_runtime_jvm_memory_limit{type="heap", %(queriesSelector)s} != -1)', }, - otel_with_suffixes: { + otel_old_with_suffixes: { expr: 'sum without (pool) (process_runtime_jvm_memory_limit_bytes{type="heap", %(queriesSelector)s} != -1)', }, + otel_with_suffixes: { + expr: 'sum without (jvm_memory_pool_name) (jvm_memory_limit_bytes{jvm_memory_type="heap", %(queriesSelector)s} != -1)', + }, prometheus_old: { expr: 'sum without (id) (jvm_memory_bytes_max{area="heap", %(queriesSelector)s} != -1)', }, @@ -83,15 +97,21 @@ function(this) sources: { //spring java_micrometer: { + expr: 'sum without (id) (jvm_memory_used{area="nonheap", %(queriesSelector)s})', + }, + java_micrometer_with_suffixes: { expr: 'sum without (id) (jvm_memory_used_bytes{area="nonheap", %(queriesSelector)s})', }, - prometheus: self.java_micrometer, - otel: { + prometheus: self.java_micrometer_with_suffixes, + otel_old: { expr: 'sum without (pool) (process_runtime_jvm_memory_usage{type="non_heap", %(queriesSelector)s})', }, - otel_with_suffixes: { + otel_old_with_suffixes: { expr: 'sum without (pool) (process_runtime_jvm_memory_usage_bytes{type="non_heap", %(queriesSelector)s})', }, + otel_with_suffixes: { + expr: 'sum without (jvm_memory_pool_name) (jvm_memory_used_bytes{jvm_memory_type="non_heap", %(queriesSelector)s})', + }, prometheus_old: { expr: 'sum without (id) (jvm_memory_bytes_used{area="nonheap", %(queriesSelector)s})', }, @@ -107,15 +127,21 @@ function(this) unit: 'bytes', sources: { java_micrometer: { + expr: 'sum without (id) (jvm_memory_max{area="nonheap", %(queriesSelector)s} != -1)', + }, + java_micrometer_with_suffixes: { expr: 'sum without (id) (jvm_memory_max_bytes{area="nonheap", %(queriesSelector)s} != -1)', }, - prometheus: self.java_micrometer, - otel: { + prometheus: self.java_micrometer_with_suffixes, + otel_old: { expr: 'sum without (pool) (process_runtime_jvm_memory_limit{type="non_heap", %(queriesSelector)s} != -1)', }, - otel_with_suffixes: { + otel_old_with_suffixes: { expr: 'sum without (pool) (process_runtime_jvm_memory_limit_bytes{type="non_heap", %(queriesSelector)s} != -1)', }, + otel_with_suffixes: { + expr: 'sum without (jvm_memory_pool_name) (jvm_memory_limit_bytes{jvm_memory_type="non_heap", %(queriesSelector)s} != -1)', + }, prometheus_old: { expr: 'sum without (id) (jvm_memory_bytes_max{area="nonheap", %(queriesSelector)s} != -1)', }, @@ -133,15 +159,21 @@ function(this) unit: 'bytes', sources: { java_micrometer: { + expr: 'sum without (id) (jvm_memory_committed{area="heap", %(queriesSelector)s})', + }, + java_micrometer_with_suffixes: { expr: 'sum without (id) (jvm_memory_committed_bytes{area="heap", %(queriesSelector)s})', }, - prometheus: self.java_micrometer, - otel: { + prometheus: self.java_micrometer_with_suffixes, + otel_old: { expr: 'sum without (pool) (process_runtime_jvm_memory_committed{type="heap", %(queriesSelector)s})', }, - otel_with_suffixes: { + otel_old_with_suffixes: { expr: 'sum without (pool) (process_runtime_jvm_memory_committed_bytes{type="heap", %(queriesSelector)s})', }, + otel_with_suffixes: { + expr: 'sum without (jvm_memory_pool_name) (jvm_memory_committed_bytes{jvm_memory_type="heap", %(queriesSelector)s})', + }, prometheus_old: { expr: 'sum without (id) (jvm_memory_bytes_committed{area="heap", %(queriesSelector)s})', }, @@ -159,15 +191,21 @@ function(this) unit: 'bytes', sources: { java_micrometer: { + expr: 'sum without (id) (jvm_memory_committed{area="nonheap", %(queriesSelector)s})', + }, + java_micrometer_with_suffixes: { expr: 'sum without (id) (jvm_memory_committed_bytes{area="nonheap", %(queriesSelector)s})', }, - prometheus: self.java_micrometer, - otel: { + prometheus: self.java_micrometer_with_suffixes, + otel_old: { expr: 'sum without (pool) (process_runtime_jvm_memory_committed{type="non_heap", %(queriesSelector)s})', }, - otel_with_suffixes: { + otel_old_with_suffixes: { expr: 'sum without (pool) (process_runtime_jvm_memory_committed_bytes{type="non_heap", %(queriesSelector)s})', }, + otel_with_suffixes: { + expr: 'sum without (jvm_memory_pool_name) (jvm_memory_committed_bytes{jvm_memory_type="non_heap", %(queriesSelector)s})', + }, prometheus_old: { expr: 'sum without (id) (jvm_memory_bytes_committed{area="nonheap", %(queriesSelector)s})', }, diff --git a/jvm-observ-lib/signals/threads.libsonnet b/jvm-observ-lib/signals/threads.libsonnet index 99e5eb68a..09420c282 100644 --- a/jvm-observ-lib/signals/threads.libsonnet +++ b/jvm-observ-lib/signals/threads.libsonnet @@ -8,10 +8,12 @@ function(this) aggLevel: 'group', aggFunction: 'avg', discoveryMetric: { - java_micrometer: 'jvm_threads_live_threads', // https://github.com/micrometer-metrics/micrometer/blob/main/micrometer-core/src/main/java/io/micrometer/core/instrument/binder/jvm/JvmThreadMetrics.java + java_micrometer: 'jvm_threads_live', // https://github.com/micrometer-metrics/micrometer/blob/main/micrometer-core/src/main/java/io/micrometer/core/instrument/binder/jvm/JvmThreadMetrics.java + java_micrometer_with_suffixes: 'jvm_threads_live_threads', // https://github.com/micrometer-metrics/micrometer/blob/main/micrometer-core/src/main/java/io/micrometer/core/instrument/binder/jvm/JvmThreadMetrics.java prometheus: 'jvm_threads_current', // https://prometheus.github.io/client_java/instrumentation/jvm/#jvm-memory-metrics - otel: 'process_runtime_jvm_threads_count', - otel_with_suffixes: self.otel, + otel_old: 'process_runtime_jvm_threads_count', + otel_old_with_suffixes: self.otel_old, + otel_with_suffixes: 'jvm_thread_count', prometheus_old: 'jvm_threads_current', jmx_exporter: 'java_lang_threading_threadcount', }, @@ -23,15 +25,21 @@ function(this) unit: 'short', sources: { java_micrometer: { + expr: 'jvm_threads_live{%(queriesSelector)s}', + }, + java_micrometer_with_suffixes: { expr: 'jvm_threads_live_threads{%(queriesSelector)s}', }, prometheus: { expr: 'jvm_threads_current{%(queriesSelector)s}', }, - otel: { + otel_old: { expr: 'sum without (daemon) (process_runtime_jvm_threads_count{%(queriesSelector)s})', }, - otel_with_suffixes: self.otel, + otel_old_with_suffixes: self.otel_old, + otel_with_suffixes: { + expr: 'sum without (jvm_thread_daemon) (jvm_thread_count{%(queriesSelector)s})', + }, prometheus_old: self.prometheus, jmx_exporter: { expr: 'java_lang_threading_threadcount{%(queriesSelector)s}', @@ -45,15 +53,21 @@ function(this) unit: 'short', sources: { java_micrometer: { + expr: 'jvm_threads_daemon{%(queriesSelector)s}', + }, + java_micrometer_with_suffixes: { expr: 'jvm_threads_daemon_threads{%(queriesSelector)s}', }, prometheus: { expr: 'jvm_threads_daemon{%(queriesSelector)s}', }, - otel: { + otel_old: { expr: 'process_runtime_jvm_threads_count{daemon="true", %(queriesSelector)s}', }, - otel_with_suffixes: self.otel, + otel_old_with_suffixes: self.otel_old, + otel_with_suffixes: { + expr: 'jvm_thread_count{jvm_thread_daemon="true", %(queriesSelector)s}', + }, prometheus_old: self.prometheus, jmx_exporter: { expr: 'java_lang_threading_daemonthreadcount{%(queriesSelector)s}', @@ -70,6 +84,9 @@ function(this) optional: true, sources: { java_micrometer: { + expr: 'jvm_threads_peak{%(queriesSelector)s}', + }, + java_micrometer_with_suffixes: { expr: 'jvm_threads_peak_threads{%(queriesSelector)s}', }, prometheus: { @@ -102,6 +119,10 @@ function(this) optional: true, sources: { java_micrometer: { + expr: 'sum by (state, %(agg)s) (jvm_threads_states{%(queriesSelector)s})', + legendCustomTemplate: '{{ state }}', + }, + java_micrometer_with_suffixes: { expr: 'sum by (state, %(agg)s) (jvm_threads_states_threads{%(queriesSelector)s})', legendCustomTemplate: '{{ state }}', }, @@ -111,7 +132,11 @@ function(this) legendCustomTemplate: '{{ state }}', }, prometheus_old: self.prometheus, - // otel: { + otel_with_suffixes: { + expr: 'sum by (jvm_thread_state, jvm_thread_daemon, %(agg)s) (jvm_thread_count{%(queriesSelector)s})', + legendCustomTemplate: '{{ jvm_thread_state }}(daemon={{jvm_thread_daemon}})', + }, + // otel_old: { // expr: '?{%(queriesSelector)s}', // }, },