diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..226a86b --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,6 @@ +--- +repos: + - repo: "https://github.com/adrienverge/yamllint" + rev: "v1.35.1" + hooks: + - id: "yamllint" diff --git a/observability/simple-datadog/.gitignore b/observability/simple-datadog/.gitignore new file mode 100644 index 0000000..4c49bd7 --- /dev/null +++ b/observability/simple-datadog/.gitignore @@ -0,0 +1 @@ +.env diff --git a/observability/simple-datadog/README.md b/observability/simple-datadog/README.md new file mode 100644 index 0000000..9f4a309 --- /dev/null +++ b/observability/simple-datadog/README.md @@ -0,0 +1,60 @@ +# Spicedb Observability with the Datadog Agent + +## Overview +This is a repository that demonstrates a configuration of SpiceDB and the Datadog Agent +that supports sending metrics and traces to Datadog. This is not the only valid configuration +and should be adapted to your use case. + +The metrics produced in this configuration are submitted as custom metrics. We're actively working +on an official SpiceDB integration that would make the metrics into standard metrics and simplify +setup and configuration. + +### Running in Production +A "real" deployment would use a container runtime of some sort. One approach would be to +run the datadog agent as a sidecar; another would be to run a set of agents using +the [Datadog Operator](https://docs.datadoghq.com/getting_started/containers/datadog_operator/) +and then point them at your SpiceDB instances using [annotations](https://docs.datadoghq.com/containers/kubernetes/integrations). +This repository is only intended to communicate the agent check configuration +and the required SpiceDB configuration. + +## Running this repo +``` +mv placeholder.env .env +``` + +Define your `DD_API_KEY` in the env file. + +Run `docker compose up`. + +### Thumper +This is an internal load-testing tool that we built a while back. We use it in this project to +exercise gRPC endpoints so that there are traces and metrics to look at. + +## The Dashboard +This is a preview of the dashboard that will be bundled with the SpiceDB Community integration. +It shows throughput, latency, and some basic node CPU and memory metrics. Note that the CPU and memory +metrics may be missing context from the container runtime environment, such as limits provided by kubernetes. + +Also note that the dashboard uses the metrics exported by SpiceDB as histogram metrics, which Datadog then internally +converts to its distribution-style metrics. There's likely some loss in resolution as a result; if this is a concern, +and 100% of traces are being collected, it may make more sense to make the latency graphs reference the trace +distribution supplied by Datadog. + +To use the dashboard, grab `spicedb-dashboard.json` and import it into Datadog. + +## Tracing +SpiceDB supports OTLP export of traces. This is configured in the environment variables in `docker-compose.yml` on +the `datadog` and `spicedb` services. Traces are pushed by SpiceDB to the Datadog agent via its OTLP endpoint, +and then the agent forwards them to Datadog. + +## Metrics +SpiceDB exposes a Prometheus metrics endpoint on port 9090 by default. This can be scraped by the Datadog Agent +using its Openmetrics integration, which is compatible with the Prometheus metrics format. The configuration is +visible in `conf.d/openmetrics.d/conf.yaml`. + +## Logs +SpiceDB writes structured JSON logs to stdout, which can be collected through your normal log collection mechanisms. + +### All Available Metrics +The configuration in `conf.d/openmetrics.d/conf.yaml` currently only includes those metrics required to drive the dashboard. +If additional metrics are desired, their names and descriptions can be found in `all_metrics.txt`. diff --git a/observability/simple-datadog/all_metrics.txt b/observability/simple-datadog/all_metrics.txt new file mode 100644 index 0000000..23bfe67 --- /dev/null +++ b/observability/simple-datadog/all_metrics.txt @@ -0,0 +1,788 @@ +# HELP crdb_client_resets cockroachdb client-side tx reset distribution +# TYPE crdb_client_resets histogram +crdb_client_resets_bucket{le="0"} 0 +crdb_client_resets_bucket{le="1"} 0 +crdb_client_resets_bucket{le="2"} 0 +crdb_client_resets_bucket{le="5"} 0 +crdb_client_resets_bucket{le="10"} 0 +crdb_client_resets_bucket{le="20"} 0 +crdb_client_resets_bucket{le="50"} 0 +crdb_client_resets_bucket{le="+Inf"} 0 +crdb_client_resets_sum 0 +crdb_client_resets_count 0 +# HELP crdb_healthy_nodes the number of healthy crdb nodes detected by spicedb +# TYPE crdb_healthy_nodes gauge +crdb_healthy_nodes 0 +# HELP go_cgo_go_to_c_calls_calls_total Count of calls made from Go to C by the current process. Sourced from /cgo/go-to-c-calls:calls +# TYPE go_cgo_go_to_c_calls_calls_total counter +go_cgo_go_to_c_calls_calls_total 0 +# HELP go_cpu_classes_gc_mark_assist_cpu_seconds_total Estimated total CPU time goroutines spent performing GC tasks to assist the GC and prevent it from falling behind the application. This metric is an overestimate, and not directly comparable to system CPU time measurements. Compare only with other /cpu/classes metrics. Sourced from /cpu/classes/gc/mark/assist:cpu-seconds +# TYPE go_cpu_classes_gc_mark_assist_cpu_seconds_total counter +go_cpu_classes_gc_mark_assist_cpu_seconds_total 0.001753435 +# HELP go_cpu_classes_gc_mark_dedicated_cpu_seconds_total Estimated total CPU time spent performing GC tasks on processors (as defined by GOMAXPROCS) dedicated to those tasks. This metric is an overestimate, and not directly comparable to system CPU time measurements. Compare only with other /cpu/classes metrics. Sourced from /cpu/classes/gc/mark/dedicated:cpu-seconds +# TYPE go_cpu_classes_gc_mark_dedicated_cpu_seconds_total counter +go_cpu_classes_gc_mark_dedicated_cpu_seconds_total 0.03961781 +# HELP go_cpu_classes_gc_mark_idle_cpu_seconds_total Estimated total CPU time spent performing GC tasks on spare CPU resources that the Go scheduler could not otherwise find a use for. This should be subtracted from the total GC CPU time to obtain a measure of compulsory GC CPU time. This metric is an overestimate, and not directly comparable to system CPU time measurements. Compare only with other /cpu/classes metrics. Sourced from /cpu/classes/gc/mark/idle:cpu-seconds +# TYPE go_cpu_classes_gc_mark_idle_cpu_seconds_total counter +go_cpu_classes_gc_mark_idle_cpu_seconds_total 0.031616581 +# HELP go_cpu_classes_gc_pause_cpu_seconds_total Estimated total CPU time spent with the application paused by the GC. Even if only one thread is running during the pause, this is computed as GOMAXPROCS times the pause latency because nothing else can be executing. This is the exact sum of samples in /sched/pauses/total/gc:seconds if each sample is multiplied by GOMAXPROCS at the time it is taken. This metric is an overestimate, and not directly comparable to system CPU time measurements. Compare only with other /cpu/classes metrics. Sourced from /cpu/classes/gc/pause:cpu-seconds +# TYPE go_cpu_classes_gc_pause_cpu_seconds_total counter +go_cpu_classes_gc_pause_cpu_seconds_total 0.007634338 +# HELP go_cpu_classes_gc_total_cpu_seconds_total Estimated total CPU time spent performing GC tasks. This metric is an overestimate, and not directly comparable to system CPU time measurements. Compare only with other /cpu/classes metrics. Sum of all metrics in /cpu/classes/gc. Sourced from /cpu/classes/gc/total:cpu-seconds +# TYPE go_cpu_classes_gc_total_cpu_seconds_total counter +go_cpu_classes_gc_total_cpu_seconds_total 0.080622164 +# HELP go_cpu_classes_idle_cpu_seconds_total Estimated total available CPU time not spent executing any Go or Go runtime code. In other words, the part of /cpu/classes/total:cpu-seconds that was unused. This metric is an overestimate, and not directly comparable to system CPU time measurements. Compare only with other /cpu/classes metrics. Sourced from /cpu/classes/idle:cpu-seconds +# TYPE go_cpu_classes_idle_cpu_seconds_total counter +go_cpu_classes_idle_cpu_seconds_total 2508.887356225 +# HELP go_cpu_classes_scavenge_assist_cpu_seconds_total Estimated total CPU time spent returning unused memory to the underlying platform in response eagerly in response to memory pressure. This metric is an overestimate, and not directly comparable to system CPU time measurements. Compare only with other /cpu/classes metrics. Sourced from /cpu/classes/scavenge/assist:cpu-seconds +# TYPE go_cpu_classes_scavenge_assist_cpu_seconds_total counter +go_cpu_classes_scavenge_assist_cpu_seconds_total 2.67e-07 +# HELP go_cpu_classes_scavenge_background_cpu_seconds_total Estimated total CPU time spent performing background tasks to return unused memory to the underlying platform. This metric is an overestimate, and not directly comparable to system CPU time measurements. Compare only with other /cpu/classes metrics. Sourced from /cpu/classes/scavenge/background:cpu-seconds +# TYPE go_cpu_classes_scavenge_background_cpu_seconds_total counter +go_cpu_classes_scavenge_background_cpu_seconds_total 1.06e-07 +# HELP go_cpu_classes_scavenge_total_cpu_seconds_total Estimated total CPU time spent performing tasks that return unused memory to the underlying platform. This metric is an overestimate, and not directly comparable to system CPU time measurements. Compare only with other /cpu/classes metrics. Sum of all metrics in /cpu/classes/scavenge. Sourced from /cpu/classes/scavenge/total:cpu-seconds +# TYPE go_cpu_classes_scavenge_total_cpu_seconds_total counter +go_cpu_classes_scavenge_total_cpu_seconds_total 3.73e-07 +# HELP go_cpu_classes_total_cpu_seconds_total Estimated total available CPU time for user Go code or the Go runtime, as defined by GOMAXPROCS. In other words, GOMAXPROCS integrated over the wall-clock duration this process has been executing for. This metric is an overestimate, and not directly comparable to system CPU time measurements. Compare only with other /cpu/classes metrics. Sum of all metrics in /cpu/classes. Sourced from /cpu/classes/total:cpu-seconds +# TYPE go_cpu_classes_total_cpu_seconds_total counter +go_cpu_classes_total_cpu_seconds_total 2509.166712672 +# HELP go_cpu_classes_user_cpu_seconds_total Estimated total CPU time spent running user Go code. This may also include some small amount of time spent in the Go runtime. This metric is an overestimate, and not directly comparable to system CPU time measurements. Compare only with other /cpu/classes metrics. Sourced from /cpu/classes/user:cpu-seconds +# TYPE go_cpu_classes_user_cpu_seconds_total counter +go_cpu_classes_user_cpu_seconds_total 0.19873391 +# HELP go_gc_cycles_automatic_gc_cycles_total Count of completed GC cycles generated by the Go runtime. Sourced from /gc/cycles/automatic:gc-cycles +# TYPE go_gc_cycles_automatic_gc_cycles_total counter +go_gc_cycles_automatic_gc_cycles_total 7 +# HELP go_gc_cycles_forced_gc_cycles_total Count of completed GC cycles forced by the application. Sourced from /gc/cycles/forced:gc-cycles +# TYPE go_gc_cycles_forced_gc_cycles_total counter +go_gc_cycles_forced_gc_cycles_total 0 +# HELP go_gc_cycles_total_gc_cycles_total Count of all completed GC cycles. Sourced from /gc/cycles/total:gc-cycles +# TYPE go_gc_cycles_total_gc_cycles_total counter +go_gc_cycles_total_gc_cycles_total 7 +# HELP go_gc_duration_seconds A summary of the wall-time pause (stop-the-world) duration in garbage collection cycles. +# TYPE go_gc_duration_seconds summary +go_gc_duration_seconds{quantile="0"} 2.2494e-05 +go_gc_duration_seconds{quantile="0.25"} 5.1619e-05 +go_gc_duration_seconds{quantile="0.5"} 6.4489e-05 +go_gc_duration_seconds{quantile="0.75"} 0.0001829 +go_gc_duration_seconds{quantile="1"} 0.000200723 +go_gc_duration_seconds_sum 0.000718186 +go_gc_duration_seconds_count 7 +# HELP go_gc_gogc_percent Heap size target percentage configured by the user, otherwise 100. This value is set by the GOGC environment variable, and the runtime/debug.SetGCPercent function. Sourced from /gc/gogc:percent +# TYPE go_gc_gogc_percent gauge +go_gc_gogc_percent 100 +# HELP go_gc_gomemlimit_bytes Go runtime memory limit configured by the user, otherwise math.MaxInt64. This value is set by the GOMEMLIMIT environment variable, and the runtime/debug.SetMemoryLimit function. Sourced from /gc/gomemlimit:bytes +# TYPE go_gc_gomemlimit_bytes gauge +go_gc_gomemlimit_bytes 3.28493056e+10 +# HELP go_gc_heap_allocs_by_size_bytes Distribution of heap allocations by approximate size. Bucket counts increase monotonically. Note that this does not include tiny objects as defined by /gc/heap/tiny/allocs:objects, only tiny blocks. Sourced from /gc/heap/allocs-by-size:bytes +# TYPE go_gc_heap_allocs_by_size_bytes histogram +go_gc_heap_allocs_by_size_bytes_bucket{le="8.999999999999998"} 5419 +go_gc_heap_allocs_by_size_bytes_bucket{le="24.999999999999996"} 112324 +go_gc_heap_allocs_by_size_bytes_bucket{le="64.99999999999999"} 202127 +go_gc_heap_allocs_by_size_bytes_bucket{le="144.99999999999997"} 263918 +go_gc_heap_allocs_by_size_bytes_bucket{le="320.99999999999994"} 274302 +go_gc_heap_allocs_by_size_bytes_bucket{le="704.9999999999999"} 276913 +go_gc_heap_allocs_by_size_bytes_bucket{le="1536.9999999999998"} 278365 +go_gc_heap_allocs_by_size_bytes_bucket{le="3200.9999999999995"} 278992 +go_gc_heap_allocs_by_size_bytes_bucket{le="6528.999999999999"} 279344 +go_gc_heap_allocs_by_size_bytes_bucket{le="13568.999999999998"} 279499 +go_gc_heap_allocs_by_size_bytes_bucket{le="27264.999999999996"} 279555 +go_gc_heap_allocs_by_size_bytes_bucket{le="+Inf"} 279636 +go_gc_heap_allocs_by_size_bytes_sum 3.2286016e+07 +go_gc_heap_allocs_by_size_bytes_count 279636 +# HELP go_gc_heap_allocs_bytes_total Cumulative sum of memory allocated to the heap by the application. Sourced from /gc/heap/allocs:bytes +# TYPE go_gc_heap_allocs_bytes_total counter +go_gc_heap_allocs_bytes_total 3.2286016e+07 +# HELP go_gc_heap_allocs_objects_total Cumulative count of heap allocations triggered by the application. Note that this does not include tiny objects as defined by /gc/heap/tiny/allocs:objects, only tiny blocks. Sourced from /gc/heap/allocs:objects +# TYPE go_gc_heap_allocs_objects_total counter +go_gc_heap_allocs_objects_total 279636 +# HELP go_gc_heap_frees_by_size_bytes Distribution of freed heap allocations by approximate size. Bucket counts increase monotonically. Note that this does not include tiny objects as defined by /gc/heap/tiny/allocs:objects, only tiny blocks. Sourced from /gc/heap/frees-by-size:bytes +# TYPE go_gc_heap_frees_by_size_bytes histogram +go_gc_heap_frees_by_size_bytes_bucket{le="8.999999999999998"} 3865 +go_gc_heap_frees_by_size_bytes_bucket{le="24.999999999999996"} 64719 +go_gc_heap_frees_by_size_bytes_bucket{le="64.99999999999999"} 107856 +go_gc_heap_frees_by_size_bytes_bucket{le="144.99999999999997"} 136336 +go_gc_heap_frees_by_size_bytes_bucket{le="320.99999999999994"} 140909 +go_gc_heap_frees_by_size_bytes_bucket{le="704.9999999999999"} 142134 +go_gc_heap_frees_by_size_bytes_bucket{le="1536.9999999999998"} 142975 +go_gc_heap_frees_by_size_bytes_bucket{le="3200.9999999999995"} 143259 +go_gc_heap_frees_by_size_bytes_bucket{le="6528.999999999999"} 143435 +go_gc_heap_frees_by_size_bytes_bucket{le="13568.999999999998"} 143498 +go_gc_heap_frees_by_size_bytes_bucket{le="27264.999999999996"} 143527 +go_gc_heap_frees_by_size_bytes_bucket{le="+Inf"} 143586 +go_gc_heap_frees_by_size_bytes_sum 1.7642992e+07 +go_gc_heap_frees_by_size_bytes_count 143586 +# HELP go_gc_heap_frees_bytes_total Cumulative sum of heap memory freed by the garbage collector. Sourced from /gc/heap/frees:bytes +# TYPE go_gc_heap_frees_bytes_total counter +go_gc_heap_frees_bytes_total 1.7642992e+07 +# HELP go_gc_heap_frees_objects_total Cumulative count of heap allocations whose storage was freed by the garbage collector. Note that this does not include tiny objects as defined by /gc/heap/tiny/allocs:objects, only tiny blocks. Sourced from /gc/heap/frees:objects +# TYPE go_gc_heap_frees_objects_total counter +go_gc_heap_frees_objects_total 143586 +# HELP go_gc_heap_goal_bytes Heap size target for the end of the GC cycle. Sourced from /gc/heap/goal:bytes +# TYPE go_gc_heap_goal_bytes gauge +go_gc_heap_goal_bytes 2.332124e+07 +# HELP go_gc_heap_live_bytes Heap memory occupied by live objects that were marked by the previous GC. Sourced from /gc/heap/live:bytes +# TYPE go_gc_heap_live_bytes gauge +go_gc_heap_live_bytes 1.1260352e+07 +# HELP go_gc_heap_objects_objects Number of objects, live or unswept, occupying heap memory. Sourced from /gc/heap/objects:objects +# TYPE go_gc_heap_objects_objects gauge +go_gc_heap_objects_objects 136050 +# HELP go_gc_heap_tiny_allocs_objects_total Count of small allocations that are packed together into blocks. These allocations are counted separately from other allocations because each individual allocation is not tracked by the runtime, only their block. Each block is already accounted for in allocs-by-size and frees-by-size. Sourced from /gc/heap/tiny/allocs:objects +# TYPE go_gc_heap_tiny_allocs_objects_total counter +go_gc_heap_tiny_allocs_objects_total 36018 +# HELP go_gc_limiter_last_enabled_gc_cycle GC cycle the last time the GC CPU limiter was enabled. This metric is useful for diagnosing the root cause of an out-of-memory error, because the limiter trades memory for CPU time when the GC's CPU time gets too high. This is most likely to occur with use of SetMemoryLimit. The first GC cycle is cycle 1, so a value of 0 indicates that it was never enabled. Sourced from /gc/limiter/last-enabled:gc-cycle +# TYPE go_gc_limiter_last_enabled_gc_cycle gauge +go_gc_limiter_last_enabled_gc_cycle 0 +# HELP go_gc_pauses_seconds Deprecated. Prefer the identical /sched/pauses/total/gc:seconds. Sourced from /gc/pauses:seconds +# TYPE go_gc_pauses_seconds histogram +go_gc_pauses_seconds_bucket{le="6.399999999999999e-08"} 0 +go_gc_pauses_seconds_bucket{le="6.399999999999999e-07"} 0 +go_gc_pauses_seconds_bucket{le="7.167999999999999e-06"} 2 +go_gc_pauses_seconds_bucket{le="8.191999999999999e-05"} 11 +go_gc_pauses_seconds_bucket{le="0.0009175039999999999"} 14 +go_gc_pauses_seconds_bucket{le="0.010485759999999998"} 14 +go_gc_pauses_seconds_bucket{le="0.11744051199999998"} 14 +go_gc_pauses_seconds_bucket{le="+Inf"} 14 +go_gc_pauses_seconds_sum 0.00031155200000000004 +go_gc_pauses_seconds_count 14 +# HELP go_gc_scan_globals_bytes The total amount of global variable space that is scannable. Sourced from /gc/scan/globals:bytes +# TYPE go_gc_scan_globals_bytes gauge +go_gc_scan_globals_bytes 764864 +# HELP go_gc_scan_heap_bytes The total amount of heap space that is scannable. Sourced from /gc/scan/heap:bytes +# TYPE go_gc_scan_heap_bytes gauge +go_gc_scan_heap_bytes 1.0542576e+07 +# HELP go_gc_scan_stack_bytes The number of bytes of stack that were scanned last GC cycle. Sourced from /gc/scan/stack:bytes +# TYPE go_gc_scan_stack_bytes gauge +go_gc_scan_stack_bytes 35672 +# HELP go_gc_scan_total_bytes The total amount space that is scannable. Sum of all metrics in /gc/scan. Sourced from /gc/scan/total:bytes +# TYPE go_gc_scan_total_bytes gauge +go_gc_scan_total_bytes 1.1343112e+07 +# HELP go_gc_stack_starting_size_bytes The stack size of new goroutines. Sourced from /gc/stack/starting-size:bytes +# TYPE go_gc_stack_starting_size_bytes gauge +go_gc_stack_starting_size_bytes 2048 +# HELP go_godebug_non_default_behavior_asynctimerchan_events_total The number of non-default behaviors executed by the time package due to a non-default GODEBUG=asynctimerchan=... setting. Sourced from /godebug/non-default-behavior/asynctimerchan:events +# TYPE go_godebug_non_default_behavior_asynctimerchan_events_total counter +go_godebug_non_default_behavior_asynctimerchan_events_total 39 +# HELP go_godebug_non_default_behavior_execerrdot_events_total The number of non-default behaviors executed by the os/exec package due to a non-default GODEBUG=execerrdot=... setting. Sourced from /godebug/non-default-behavior/execerrdot:events +# TYPE go_godebug_non_default_behavior_execerrdot_events_total counter +go_godebug_non_default_behavior_execerrdot_events_total 0 +# HELP go_godebug_non_default_behavior_gocachehash_events_total The number of non-default behaviors executed by the cmd/go package due to a non-default GODEBUG=gocachehash=... setting. Sourced from /godebug/non-default-behavior/gocachehash:events +# TYPE go_godebug_non_default_behavior_gocachehash_events_total counter +go_godebug_non_default_behavior_gocachehash_events_total 0 +# HELP go_godebug_non_default_behavior_gocachetest_events_total The number of non-default behaviors executed by the cmd/go package due to a non-default GODEBUG=gocachetest=... setting. Sourced from /godebug/non-default-behavior/gocachetest:events +# TYPE go_godebug_non_default_behavior_gocachetest_events_total counter +go_godebug_non_default_behavior_gocachetest_events_total 0 +# HELP go_godebug_non_default_behavior_gocacheverify_events_total The number of non-default behaviors executed by the cmd/go package due to a non-default GODEBUG=gocacheverify=... setting. Sourced from /godebug/non-default-behavior/gocacheverify:events +# TYPE go_godebug_non_default_behavior_gocacheverify_events_total counter +go_godebug_non_default_behavior_gocacheverify_events_total 0 +# HELP go_godebug_non_default_behavior_gotypesalias_events_total The number of non-default behaviors executed by the go/types package due to a non-default GODEBUG=gotypesalias=... setting. Sourced from /godebug/non-default-behavior/gotypesalias:events +# TYPE go_godebug_non_default_behavior_gotypesalias_events_total counter +go_godebug_non_default_behavior_gotypesalias_events_total 0 +# HELP go_godebug_non_default_behavior_http2client_events_total The number of non-default behaviors executed by the net/http package due to a non-default GODEBUG=http2client=... setting. Sourced from /godebug/non-default-behavior/http2client:events +# TYPE go_godebug_non_default_behavior_http2client_events_total counter +go_godebug_non_default_behavior_http2client_events_total 0 +# HELP go_godebug_non_default_behavior_http2server_events_total The number of non-default behaviors executed by the net/http package due to a non-default GODEBUG=http2server=... setting. Sourced from /godebug/non-default-behavior/http2server:events +# TYPE go_godebug_non_default_behavior_http2server_events_total counter +go_godebug_non_default_behavior_http2server_events_total 0 +# HELP go_godebug_non_default_behavior_httplaxcontentlength_events_total The number of non-default behaviors executed by the net/http package due to a non-default GODEBUG=httplaxcontentlength=... setting. Sourced from /godebug/non-default-behavior/httplaxcontentlength:events +# TYPE go_godebug_non_default_behavior_httplaxcontentlength_events_total counter +go_godebug_non_default_behavior_httplaxcontentlength_events_total 0 +# HELP go_godebug_non_default_behavior_httpmuxgo121_events_total The number of non-default behaviors executed by the net/http package due to a non-default GODEBUG=httpmuxgo121=... setting. Sourced from /godebug/non-default-behavior/httpmuxgo121:events +# TYPE go_godebug_non_default_behavior_httpmuxgo121_events_total counter +go_godebug_non_default_behavior_httpmuxgo121_events_total 0 +# HELP go_godebug_non_default_behavior_httpservecontentkeepheaders_events_total The number of non-default behaviors executed by the net/http package due to a non-default GODEBUG=httpservecontentkeepheaders=... setting. Sourced from /godebug/non-default-behavior/httpservecontentkeepheaders:events +# TYPE go_godebug_non_default_behavior_httpservecontentkeepheaders_events_total counter +go_godebug_non_default_behavior_httpservecontentkeepheaders_events_total 0 +# HELP go_godebug_non_default_behavior_installgoroot_events_total The number of non-default behaviors executed by the go/build package due to a non-default GODEBUG=installgoroot=... setting. Sourced from /godebug/non-default-behavior/installgoroot:events +# TYPE go_godebug_non_default_behavior_installgoroot_events_total counter +go_godebug_non_default_behavior_installgoroot_events_total 0 +# HELP go_godebug_non_default_behavior_multipartmaxheaders_events_total The number of non-default behaviors executed by the mime/multipart package due to a non-default GODEBUG=multipartmaxheaders=... setting. Sourced from /godebug/non-default-behavior/multipartmaxheaders:events +# TYPE go_godebug_non_default_behavior_multipartmaxheaders_events_total counter +go_godebug_non_default_behavior_multipartmaxheaders_events_total 0 +# HELP go_godebug_non_default_behavior_multipartmaxparts_events_total The number of non-default behaviors executed by the mime/multipart package due to a non-default GODEBUG=multipartmaxparts=... setting. Sourced from /godebug/non-default-behavior/multipartmaxparts:events +# TYPE go_godebug_non_default_behavior_multipartmaxparts_events_total counter +go_godebug_non_default_behavior_multipartmaxparts_events_total 0 +# HELP go_godebug_non_default_behavior_multipathtcp_events_total The number of non-default behaviors executed by the net package due to a non-default GODEBUG=multipathtcp=... setting. Sourced from /godebug/non-default-behavior/multipathtcp:events +# TYPE go_godebug_non_default_behavior_multipathtcp_events_total counter +go_godebug_non_default_behavior_multipathtcp_events_total 0 +# HELP go_godebug_non_default_behavior_netedns0_events_total The number of non-default behaviors executed by the net package due to a non-default GODEBUG=netedns0=... setting. Sourced from /godebug/non-default-behavior/netedns0:events +# TYPE go_godebug_non_default_behavior_netedns0_events_total counter +go_godebug_non_default_behavior_netedns0_events_total 0 +# HELP go_godebug_non_default_behavior_panicnil_events_total The number of non-default behaviors executed by the runtime package due to a non-default GODEBUG=panicnil=... setting. Sourced from /godebug/non-default-behavior/panicnil:events +# TYPE go_godebug_non_default_behavior_panicnil_events_total counter +go_godebug_non_default_behavior_panicnil_events_total 0 +# HELP go_godebug_non_default_behavior_randautoseed_events_total The number of non-default behaviors executed by the math/rand package due to a non-default GODEBUG=randautoseed=... setting. Sourced from /godebug/non-default-behavior/randautoseed:events +# TYPE go_godebug_non_default_behavior_randautoseed_events_total counter +go_godebug_non_default_behavior_randautoseed_events_total 0 +# HELP go_godebug_non_default_behavior_tarinsecurepath_events_total The number of non-default behaviors executed by the archive/tar package due to a non-default GODEBUG=tarinsecurepath=... setting. Sourced from /godebug/non-default-behavior/tarinsecurepath:events +# TYPE go_godebug_non_default_behavior_tarinsecurepath_events_total counter +go_godebug_non_default_behavior_tarinsecurepath_events_total 0 +# HELP go_godebug_non_default_behavior_tls10server_events_total The number of non-default behaviors executed by the crypto/tls package due to a non-default GODEBUG=tls10server=... setting. Sourced from /godebug/non-default-behavior/tls10server:events +# TYPE go_godebug_non_default_behavior_tls10server_events_total counter +go_godebug_non_default_behavior_tls10server_events_total 0 +# HELP go_godebug_non_default_behavior_tls3des_events_total The number of non-default behaviors executed by the crypto/tls package due to a non-default GODEBUG=tls3des=... setting. Sourced from /godebug/non-default-behavior/tls3des:events +# TYPE go_godebug_non_default_behavior_tls3des_events_total counter +go_godebug_non_default_behavior_tls3des_events_total 0 +# HELP go_godebug_non_default_behavior_tlsmaxrsasize_events_total The number of non-default behaviors executed by the crypto/tls package due to a non-default GODEBUG=tlsmaxrsasize=... setting. Sourced from /godebug/non-default-behavior/tlsmaxrsasize:events +# TYPE go_godebug_non_default_behavior_tlsmaxrsasize_events_total counter +go_godebug_non_default_behavior_tlsmaxrsasize_events_total 0 +# HELP go_godebug_non_default_behavior_tlsrsakex_events_total The number of non-default behaviors executed by the crypto/tls package due to a non-default GODEBUG=tlsrsakex=... setting. Sourced from /godebug/non-default-behavior/tlsrsakex:events +# TYPE go_godebug_non_default_behavior_tlsrsakex_events_total counter +go_godebug_non_default_behavior_tlsrsakex_events_total 0 +# HELP go_godebug_non_default_behavior_tlsunsafeekm_events_total The number of non-default behaviors executed by the crypto/tls package due to a non-default GODEBUG=tlsunsafeekm=... setting. Sourced from /godebug/non-default-behavior/tlsunsafeekm:events +# TYPE go_godebug_non_default_behavior_tlsunsafeekm_events_total counter +go_godebug_non_default_behavior_tlsunsafeekm_events_total 0 +# HELP go_godebug_non_default_behavior_winreadlinkvolume_events_total The number of non-default behaviors executed by the os package due to a non-default GODEBUG=winreadlinkvolume=... setting. Sourced from /godebug/non-default-behavior/winreadlinkvolume:events +# TYPE go_godebug_non_default_behavior_winreadlinkvolume_events_total counter +go_godebug_non_default_behavior_winreadlinkvolume_events_total 0 +# HELP go_godebug_non_default_behavior_winsymlink_events_total The number of non-default behaviors executed by the os package due to a non-default GODEBUG=winsymlink=... setting. Sourced from /godebug/non-default-behavior/winsymlink:events +# TYPE go_godebug_non_default_behavior_winsymlink_events_total counter +go_godebug_non_default_behavior_winsymlink_events_total 0 +# HELP go_godebug_non_default_behavior_x509keypairleaf_events_total The number of non-default behaviors executed by the crypto/tls package due to a non-default GODEBUG=x509keypairleaf=... setting. Sourced from /godebug/non-default-behavior/x509keypairleaf:events +# TYPE go_godebug_non_default_behavior_x509keypairleaf_events_total counter +go_godebug_non_default_behavior_x509keypairleaf_events_total 0 +# HELP go_godebug_non_default_behavior_x509negativeserial_events_total The number of non-default behaviors executed by the crypto/x509 package due to a non-default GODEBUG=x509negativeserial=... setting. Sourced from /godebug/non-default-behavior/x509negativeserial:events +# TYPE go_godebug_non_default_behavior_x509negativeserial_events_total counter +go_godebug_non_default_behavior_x509negativeserial_events_total 0 +# HELP go_godebug_non_default_behavior_x509sha1_events_total The number of non-default behaviors executed by the crypto/x509 package due to a non-default GODEBUG=x509sha1=... setting. Sourced from /godebug/non-default-behavior/x509sha1:events +# TYPE go_godebug_non_default_behavior_x509sha1_events_total counter +go_godebug_non_default_behavior_x509sha1_events_total 0 +# HELP go_godebug_non_default_behavior_x509usefallbackroots_events_total The number of non-default behaviors executed by the crypto/x509 package due to a non-default GODEBUG=x509usefallbackroots=... setting. Sourced from /godebug/non-default-behavior/x509usefallbackroots:events +# TYPE go_godebug_non_default_behavior_x509usefallbackroots_events_total counter +go_godebug_non_default_behavior_x509usefallbackroots_events_total 0 +# HELP go_godebug_non_default_behavior_x509usepolicies_events_total The number of non-default behaviors executed by the crypto/x509 package due to a non-default GODEBUG=x509usepolicies=... setting. Sourced from /godebug/non-default-behavior/x509usepolicies:events +# TYPE go_godebug_non_default_behavior_x509usepolicies_events_total counter +go_godebug_non_default_behavior_x509usepolicies_events_total 0 +# HELP go_godebug_non_default_behavior_zipinsecurepath_events_total The number of non-default behaviors executed by the archive/zip package due to a non-default GODEBUG=zipinsecurepath=... setting. Sourced from /godebug/non-default-behavior/zipinsecurepath:events +# TYPE go_godebug_non_default_behavior_zipinsecurepath_events_total counter +go_godebug_non_default_behavior_zipinsecurepath_events_total 0 +# HELP go_goroutines Number of goroutines that currently exist. +# TYPE go_goroutines gauge +go_goroutines 33 +# HELP go_info Information about the Go environment. +# TYPE go_info gauge +go_info{version="go1.23.1"} 1 +# HELP go_memory_classes_heap_free_bytes Memory that is completely free and eligible to be returned to the underlying system, but has not been. This metric is the runtime's estimate of free address space that is backed by physical memory. Sourced from /memory/classes/heap/free:bytes +# TYPE go_memory_classes_heap_free_bytes gauge +go_memory_classes_heap_free_bytes 3.489792e+06 +# HELP go_memory_classes_heap_objects_bytes Memory occupied by live objects and dead objects that have not yet been marked free by the garbage collector. Sourced from /memory/classes/heap/objects:bytes +# TYPE go_memory_classes_heap_objects_bytes gauge +go_memory_classes_heap_objects_bytes 1.4643024e+07 +# HELP go_memory_classes_heap_released_bytes Memory that is completely free and has been returned to the underlying system. This metric is the runtime's estimate of free address space that is still mapped into the process, but is not backed by physical memory. Sourced from /memory/classes/heap/released:bytes +# TYPE go_memory_classes_heap_released_bytes gauge +go_memory_classes_heap_released_bytes 2.392064e+06 +# HELP go_memory_classes_heap_stacks_bytes Memory allocated from the heap that is reserved for stack space, whether or not it is currently in-use. Currently, this represents all stack memory for goroutines. It also includes all OS thread stacks in non-cgo programs. Note that stacks may be allocated differently in the future, and this may change. Sourced from /memory/classes/heap/stacks:bytes +# TYPE go_memory_classes_heap_stacks_bytes gauge +go_memory_classes_heap_stacks_bytes 1.343488e+06 +# HELP go_memory_classes_heap_unused_bytes Memory that is reserved for heap objects but is not currently used to hold heap objects. Sourced from /memory/classes/heap/unused:bytes +# TYPE go_memory_classes_heap_unused_bytes gauge +go_memory_classes_heap_unused_bytes 3.297456e+06 +# HELP go_memory_classes_metadata_mcache_free_bytes Memory that is reserved for runtime mcache structures, but not in-use. Sourced from /memory/classes/metadata/mcache/free:bytes +# TYPE go_memory_classes_metadata_mcache_free_bytes gauge +go_memory_classes_metadata_mcache_free_bytes 1200 +# HELP go_memory_classes_metadata_mcache_inuse_bytes Memory that is occupied by runtime mcache structures that are currently being used. Sourced from /memory/classes/metadata/mcache/inuse:bytes +# TYPE go_memory_classes_metadata_mcache_inuse_bytes gauge +go_memory_classes_metadata_mcache_inuse_bytes 14400 +# HELP go_memory_classes_metadata_mspan_free_bytes Memory that is reserved for runtime mspan structures, but not in-use. Sourced from /memory/classes/metadata/mspan/free:bytes +# TYPE go_memory_classes_metadata_mspan_free_bytes gauge +go_memory_classes_metadata_mspan_free_bytes 1760 +# HELP go_memory_classes_metadata_mspan_inuse_bytes Memory that is occupied by runtime mspan structures that are currently being used. Sourced from /memory/classes/metadata/mspan/inuse:bytes +# TYPE go_memory_classes_metadata_mspan_inuse_bytes gauge +go_memory_classes_metadata_mspan_inuse_bytes 357280 +# HELP go_memory_classes_metadata_other_bytes Memory that is reserved for or used to hold runtime metadata. Sourced from /memory/classes/metadata/other:bytes +# TYPE go_memory_classes_metadata_other_bytes gauge +go_memory_classes_metadata_other_bytes 3.726896e+06 +# HELP go_memory_classes_os_stacks_bytes Stack memory allocated by the underlying operating system. In non-cgo programs this metric is currently zero. This may change in the future.In cgo programs this metric includes OS thread stacks allocated directly from the OS. Currently, this only accounts for one stack in c-shared and c-archive build modes, and other sources of stacks from the OS are not measured. This too may change in the future. Sourced from /memory/classes/os-stacks:bytes +# TYPE go_memory_classes_os_stacks_bytes gauge +go_memory_classes_os_stacks_bytes 0 +# HELP go_memory_classes_other_bytes Memory used by execution trace buffers, structures for debugging the runtime, finalizer and profiler specials, and more. Sourced from /memory/classes/other:bytes +# TYPE go_memory_classes_other_bytes gauge +go_memory_classes_other_bytes 2.115807e+06 +# HELP go_memory_classes_profiling_buckets_bytes Memory that is used by the stack trace hash map used for profiling. Sourced from /memory/classes/profiling/buckets:bytes +# TYPE go_memory_classes_profiling_buckets_bytes gauge +go_memory_classes_profiling_buckets_bytes 1.472137e+06 +# HELP go_memory_classes_total_bytes All memory mapped by the Go runtime into the current process as read-write. Note that this does not include memory mapped by code called via cgo or via the syscall package. Sum of all metrics in /memory/classes. Sourced from /memory/classes/total:bytes +# TYPE go_memory_classes_total_bytes gauge +go_memory_classes_total_bytes 3.2855304e+07 +# HELP go_memstats_alloc_bytes Number of bytes allocated in heap and currently in use. Equals to /memory/classes/heap/objects:bytes. +# TYPE go_memstats_alloc_bytes gauge +go_memstats_alloc_bytes 1.4643024e+07 +# HELP go_memstats_alloc_bytes_total Total number of bytes allocated in heap until now, even if released already. Equals to /gc/heap/allocs:bytes. +# TYPE go_memstats_alloc_bytes_total counter +go_memstats_alloc_bytes_total 3.2286016e+07 +# HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table. Equals to /memory/classes/profiling/buckets:bytes. +# TYPE go_memstats_buck_hash_sys_bytes gauge +go_memstats_buck_hash_sys_bytes 1.472137e+06 +# HELP go_memstats_frees_total Total number of heap objects frees. Equals to /gc/heap/frees:objects + /gc/heap/tiny/allocs:objects. +# TYPE go_memstats_frees_total counter +go_memstats_frees_total 179604 +# HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata. Equals to /memory/classes/metadata/other:bytes. +# TYPE go_memstats_gc_sys_bytes gauge +go_memstats_gc_sys_bytes 3.726896e+06 +# HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and currently in use, same as go_memstats_alloc_bytes. Equals to /memory/classes/heap/objects:bytes. +# TYPE go_memstats_heap_alloc_bytes gauge +go_memstats_heap_alloc_bytes 1.4643024e+07 +# HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used. Equals to /memory/classes/heap/released:bytes + /memory/classes/heap/free:bytes. +# TYPE go_memstats_heap_idle_bytes gauge +go_memstats_heap_idle_bytes 5.881856e+06 +# HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use. Equals to /memory/classes/heap/objects:bytes + /memory/classes/heap/unused:bytes +# TYPE go_memstats_heap_inuse_bytes gauge +go_memstats_heap_inuse_bytes 1.794048e+07 +# HELP go_memstats_heap_objects Number of currently allocated objects. Equals to /gc/heap/objects:objects. +# TYPE go_memstats_heap_objects gauge +go_memstats_heap_objects 136050 +# HELP go_memstats_heap_released_bytes Number of heap bytes released to OS. Equals to /memory/classes/heap/released:bytes. +# TYPE go_memstats_heap_released_bytes gauge +go_memstats_heap_released_bytes 2.392064e+06 +# HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system. Equals to /memory/classes/heap/objects:bytes + /memory/classes/heap/unused:bytes + /memory/classes/heap/released:bytes + /memory/classes/heap/free:bytes. +# TYPE go_memstats_heap_sys_bytes gauge +go_memstats_heap_sys_bytes 2.3822336e+07 +# HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection. +# TYPE go_memstats_last_gc_time_seconds gauge +go_memstats_last_gc_time_seconds 1.7317049496282802e+09 +# HELP go_memstats_mallocs_total Total number of heap objects allocated, both live and gc-ed. Semantically a counter version for go_memstats_heap_objects gauge. Equals to /gc/heap/allocs:objects + /gc/heap/tiny/allocs:objects. +# TYPE go_memstats_mallocs_total counter +go_memstats_mallocs_total 315654 +# HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures. Equals to /memory/classes/metadata/mcache/inuse:bytes. +# TYPE go_memstats_mcache_inuse_bytes gauge +go_memstats_mcache_inuse_bytes 14400 +# HELP go_memstats_mcache_sys_bytes Number of bytes used for mcache structures obtained from system. Equals to /memory/classes/metadata/mcache/inuse:bytes + /memory/classes/metadata/mcache/free:bytes. +# TYPE go_memstats_mcache_sys_bytes gauge +go_memstats_mcache_sys_bytes 15600 +# HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures. Equals to /memory/classes/metadata/mspan/inuse:bytes. +# TYPE go_memstats_mspan_inuse_bytes gauge +go_memstats_mspan_inuse_bytes 357280 +# HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system. Equals to /memory/classes/metadata/mspan/inuse:bytes + /memory/classes/metadata/mspan/free:bytes. +# TYPE go_memstats_mspan_sys_bytes gauge +go_memstats_mspan_sys_bytes 359040 +# HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place. Equals to /gc/heap/goal:bytes. +# TYPE go_memstats_next_gc_bytes gauge +go_memstats_next_gc_bytes 2.332124e+07 +# HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations. Equals to /memory/classes/other:bytes. +# TYPE go_memstats_other_sys_bytes gauge +go_memstats_other_sys_bytes 2.115807e+06 +# HELP go_memstats_stack_inuse_bytes Number of bytes obtained from system for stack allocator in non-CGO environments. Equals to /memory/classes/heap/stacks:bytes. +# TYPE go_memstats_stack_inuse_bytes gauge +go_memstats_stack_inuse_bytes 1.343488e+06 +# HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator. Equals to /memory/classes/heap/stacks:bytes + /memory/classes/os-stacks:bytes. +# TYPE go_memstats_stack_sys_bytes gauge +go_memstats_stack_sys_bytes 1.343488e+06 +# HELP go_memstats_sys_bytes Number of bytes obtained from system. Equals to /memory/classes/total:byte. +# TYPE go_memstats_sys_bytes gauge +go_memstats_sys_bytes 3.2855304e+07 +# HELP go_sched_gomaxprocs_threads The current runtime.GOMAXPROCS setting, or the number of operating system threads that can execute user-level Go code simultaneously. Sourced from /sched/gomaxprocs:threads +# TYPE go_sched_gomaxprocs_threads gauge +go_sched_gomaxprocs_threads 12 +# HELP go_sched_goroutines_goroutines Count of live goroutines. Sourced from /sched/goroutines:goroutines +# TYPE go_sched_goroutines_goroutines gauge +go_sched_goroutines_goroutines 33 +# HELP go_sched_latencies_seconds Distribution of the time goroutines have spent in the scheduler in a runnable state before actually running. Bucket counts increase monotonically. Sourced from /sched/latencies:seconds +# TYPE go_sched_latencies_seconds histogram +go_sched_latencies_seconds_bucket{le="6.399999999999999e-08"} 200 +go_sched_latencies_seconds_bucket{le="6.399999999999999e-07"} 245 +go_sched_latencies_seconds_bucket{le="7.167999999999999e-06"} 309 +go_sched_latencies_seconds_bucket{le="8.191999999999999e-05"} 348 +go_sched_latencies_seconds_bucket{le="0.0009175039999999999"} 353 +go_sched_latencies_seconds_bucket{le="0.010485759999999998"} 353 +go_sched_latencies_seconds_bucket{le="0.11744051199999998"} 353 +go_sched_latencies_seconds_bucket{le="+Inf"} 353 +go_sched_latencies_seconds_sum 0.000732992 +go_sched_latencies_seconds_count 353 +# HELP go_sched_pauses_stopping_gc_seconds Distribution of individual GC-related stop-the-world stopping latencies. This is the time it takes from deciding to stop the world until all Ps are stopped. This is a subset of the total GC-related stop-the-world time (/sched/pauses/total/gc:seconds). During this time, some threads may be executing. Bucket counts increase monotonically. Sourced from /sched/pauses/stopping/gc:seconds +# TYPE go_sched_pauses_stopping_gc_seconds histogram +go_sched_pauses_stopping_gc_seconds_bucket{le="6.399999999999999e-08"} 0 +go_sched_pauses_stopping_gc_seconds_bucket{le="6.399999999999999e-07"} 0 +go_sched_pauses_stopping_gc_seconds_bucket{le="7.167999999999999e-06"} 10 +go_sched_pauses_stopping_gc_seconds_bucket{le="8.191999999999999e-05"} 12 +go_sched_pauses_stopping_gc_seconds_bucket{le="0.0009175039999999999"} 14 +go_sched_pauses_stopping_gc_seconds_bucket{le="0.010485759999999998"} 14 +go_sched_pauses_stopping_gc_seconds_bucket{le="0.11744051199999998"} 14 +go_sched_pauses_stopping_gc_seconds_bucket{le="+Inf"} 14 +go_sched_pauses_stopping_gc_seconds_sum 0.000184576 +go_sched_pauses_stopping_gc_seconds_count 14 +# HELP go_sched_pauses_stopping_other_seconds Distribution of individual non-GC-related stop-the-world stopping latencies. This is the time it takes from deciding to stop the world until all Ps are stopped. This is a subset of the total non-GC-related stop-the-world time (/sched/pauses/total/other:seconds). During this time, some threads may be executing. Bucket counts increase monotonically. Sourced from /sched/pauses/stopping/other:seconds +# TYPE go_sched_pauses_stopping_other_seconds histogram +go_sched_pauses_stopping_other_seconds_bucket{le="6.399999999999999e-08"} 0 +go_sched_pauses_stopping_other_seconds_bucket{le="6.399999999999999e-07"} 0 +go_sched_pauses_stopping_other_seconds_bucket{le="7.167999999999999e-06"} 0 +go_sched_pauses_stopping_other_seconds_bucket{le="8.191999999999999e-05"} 0 +go_sched_pauses_stopping_other_seconds_bucket{le="0.0009175039999999999"} 0 +go_sched_pauses_stopping_other_seconds_bucket{le="0.010485759999999998"} 0 +go_sched_pauses_stopping_other_seconds_bucket{le="0.11744051199999998"} 0 +go_sched_pauses_stopping_other_seconds_bucket{le="+Inf"} 0 +go_sched_pauses_stopping_other_seconds_sum 0 +go_sched_pauses_stopping_other_seconds_count 0 +# HELP go_sched_pauses_total_gc_seconds Distribution of individual GC-related stop-the-world pause latencies. This is the time from deciding to stop the world until the world is started again. Some of this time is spent getting all threads to stop (this is measured directly in /sched/pauses/stopping/gc:seconds), during which some threads may still be running. Bucket counts increase monotonically. Sourced from /sched/pauses/total/gc:seconds +# TYPE go_sched_pauses_total_gc_seconds histogram +go_sched_pauses_total_gc_seconds_bucket{le="6.399999999999999e-08"} 0 +go_sched_pauses_total_gc_seconds_bucket{le="6.399999999999999e-07"} 0 +go_sched_pauses_total_gc_seconds_bucket{le="7.167999999999999e-06"} 2 +go_sched_pauses_total_gc_seconds_bucket{le="8.191999999999999e-05"} 11 +go_sched_pauses_total_gc_seconds_bucket{le="0.0009175039999999999"} 14 +go_sched_pauses_total_gc_seconds_bucket{le="0.010485759999999998"} 14 +go_sched_pauses_total_gc_seconds_bucket{le="0.11744051199999998"} 14 +go_sched_pauses_total_gc_seconds_bucket{le="+Inf"} 14 +go_sched_pauses_total_gc_seconds_sum 0.00031155200000000004 +go_sched_pauses_total_gc_seconds_count 14 +# HELP go_sched_pauses_total_other_seconds Distribution of individual non-GC-related stop-the-world pause latencies. This is the time from deciding to stop the world until the world is started again. Some of this time is spent getting all threads to stop (measured directly in /sched/pauses/stopping/other:seconds). Bucket counts increase monotonically. Sourced from /sched/pauses/total/other:seconds +# TYPE go_sched_pauses_total_other_seconds histogram +go_sched_pauses_total_other_seconds_bucket{le="6.399999999999999e-08"} 0 +go_sched_pauses_total_other_seconds_bucket{le="6.399999999999999e-07"} 0 +go_sched_pauses_total_other_seconds_bucket{le="7.167999999999999e-06"} 0 +go_sched_pauses_total_other_seconds_bucket{le="8.191999999999999e-05"} 0 +go_sched_pauses_total_other_seconds_bucket{le="0.0009175039999999999"} 0 +go_sched_pauses_total_other_seconds_bucket{le="0.010485759999999998"} 0 +go_sched_pauses_total_other_seconds_bucket{le="0.11744051199999998"} 0 +go_sched_pauses_total_other_seconds_bucket{le="+Inf"} 0 +go_sched_pauses_total_other_seconds_sum 0 +go_sched_pauses_total_other_seconds_count 0 +# HELP go_sync_mutex_wait_total_seconds_total Approximate cumulative time goroutines have spent blocked on a sync.Mutex, sync.RWMutex, or runtime-internal lock. This metric is useful for identifying global changes in lock contention. Collect a mutex or block profile using the runtime/pprof package for more detailed contention data. Sourced from /sync/mutex/wait/total:seconds +# TYPE go_sync_mutex_wait_total_seconds_total counter +go_sync_mutex_wait_total_seconds_total 0.0007877439999999999 +# HELP go_threads Number of OS threads created. +# TYPE go_threads gauge +go_threads 17 +# HELP grpc_server_handled_total Total number of RPCs completed on the server, regardless of success or failure. +# TYPE grpc_server_handled_total counter +grpc_server_handled_total{grpc_code="OK",grpc_method="ImportBulkRelationships",grpc_service="authzed.api.v1.PermissionsService",grpc_type="client_stream"} 10 +grpc_server_handled_total{grpc_code="OK",grpc_method="WriteSchema",grpc_service="authzed.api.v1.SchemaService",grpc_type="unary"} 1 +# HELP grpc_server_handling_seconds Histogram of response latency (seconds) of gRPC that had been application-level handled by the server. +# TYPE grpc_server_handling_seconds histogram +grpc_server_handling_seconds_bucket{grpc_method="ImportBulkRelationships",grpc_service="authzed.api.v1.PermissionsService",grpc_type="client_stream",le="0.001"} 0 +grpc_server_handling_seconds_bucket{grpc_method="ImportBulkRelationships",grpc_service="authzed.api.v1.PermissionsService",grpc_type="client_stream",le="0.003"} 2 +grpc_server_handling_seconds_bucket{grpc_method="ImportBulkRelationships",grpc_service="authzed.api.v1.PermissionsService",grpc_type="client_stream",le="0.006"} 8 +grpc_server_handling_seconds_bucket{grpc_method="ImportBulkRelationships",grpc_service="authzed.api.v1.PermissionsService",grpc_type="client_stream",le="0.01"} 9 +grpc_server_handling_seconds_bucket{grpc_method="ImportBulkRelationships",grpc_service="authzed.api.v1.PermissionsService",grpc_type="client_stream",le="0.018"} 10 +grpc_server_handling_seconds_bucket{grpc_method="ImportBulkRelationships",grpc_service="authzed.api.v1.PermissionsService",grpc_type="client_stream",le="0.024"} 10 +grpc_server_handling_seconds_bucket{grpc_method="ImportBulkRelationships",grpc_service="authzed.api.v1.PermissionsService",grpc_type="client_stream",le="0.032"} 10 +grpc_server_handling_seconds_bucket{grpc_method="ImportBulkRelationships",grpc_service="authzed.api.v1.PermissionsService",grpc_type="client_stream",le="0.042"} 10 +grpc_server_handling_seconds_bucket{grpc_method="ImportBulkRelationships",grpc_service="authzed.api.v1.PermissionsService",grpc_type="client_stream",le="0.056"} 10 +grpc_server_handling_seconds_bucket{grpc_method="ImportBulkRelationships",grpc_service="authzed.api.v1.PermissionsService",grpc_type="client_stream",le="0.075"} 10 +grpc_server_handling_seconds_bucket{grpc_method="ImportBulkRelationships",grpc_service="authzed.api.v1.PermissionsService",grpc_type="client_stream",le="0.1"} 10 +grpc_server_handling_seconds_bucket{grpc_method="ImportBulkRelationships",grpc_service="authzed.api.v1.PermissionsService",grpc_type="client_stream",le="0.178"} 10 +grpc_server_handling_seconds_bucket{grpc_method="ImportBulkRelationships",grpc_service="authzed.api.v1.PermissionsService",grpc_type="client_stream",le="0.316"} 10 +grpc_server_handling_seconds_bucket{grpc_method="ImportBulkRelationships",grpc_service="authzed.api.v1.PermissionsService",grpc_type="client_stream",le="0.562"} 10 +grpc_server_handling_seconds_bucket{grpc_method="ImportBulkRelationships",grpc_service="authzed.api.v1.PermissionsService",grpc_type="client_stream",le="1"} 10 +grpc_server_handling_seconds_bucket{grpc_method="ImportBulkRelationships",grpc_service="authzed.api.v1.PermissionsService",grpc_type="client_stream",le="5"} 10 +grpc_server_handling_seconds_bucket{grpc_method="ImportBulkRelationships",grpc_service="authzed.api.v1.PermissionsService",grpc_type="client_stream",le="+Inf"} 10 +grpc_server_handling_seconds_sum{grpc_method="ImportBulkRelationships",grpc_service="authzed.api.v1.PermissionsService",grpc_type="client_stream"} 0.043256006 +grpc_server_handling_seconds_count{grpc_method="ImportBulkRelationships",grpc_service="authzed.api.v1.PermissionsService",grpc_type="client_stream"} 10 +grpc_server_handling_seconds_bucket{grpc_method="WriteSchema",grpc_service="authzed.api.v1.SchemaService",grpc_type="unary",le="0.001"} 0 +grpc_server_handling_seconds_bucket{grpc_method="WriteSchema",grpc_service="authzed.api.v1.SchemaService",grpc_type="unary",le="0.003"} 0 +grpc_server_handling_seconds_bucket{grpc_method="WriteSchema",grpc_service="authzed.api.v1.SchemaService",grpc_type="unary",le="0.006"} 0 +grpc_server_handling_seconds_bucket{grpc_method="WriteSchema",grpc_service="authzed.api.v1.SchemaService",grpc_type="unary",le="0.01"} 0 +grpc_server_handling_seconds_bucket{grpc_method="WriteSchema",grpc_service="authzed.api.v1.SchemaService",grpc_type="unary",le="0.018"} 0 +grpc_server_handling_seconds_bucket{grpc_method="WriteSchema",grpc_service="authzed.api.v1.SchemaService",grpc_type="unary",le="0.024"} 0 +grpc_server_handling_seconds_bucket{grpc_method="WriteSchema",grpc_service="authzed.api.v1.SchemaService",grpc_type="unary",le="0.032"} 0 +grpc_server_handling_seconds_bucket{grpc_method="WriteSchema",grpc_service="authzed.api.v1.SchemaService",grpc_type="unary",le="0.042"} 1 +grpc_server_handling_seconds_bucket{grpc_method="WriteSchema",grpc_service="authzed.api.v1.SchemaService",grpc_type="unary",le="0.056"} 1 +grpc_server_handling_seconds_bucket{grpc_method="WriteSchema",grpc_service="authzed.api.v1.SchemaService",grpc_type="unary",le="0.075"} 1 +grpc_server_handling_seconds_bucket{grpc_method="WriteSchema",grpc_service="authzed.api.v1.SchemaService",grpc_type="unary",le="0.1"} 1 +grpc_server_handling_seconds_bucket{grpc_method="WriteSchema",grpc_service="authzed.api.v1.SchemaService",grpc_type="unary",le="0.178"} 1 +grpc_server_handling_seconds_bucket{grpc_method="WriteSchema",grpc_service="authzed.api.v1.SchemaService",grpc_type="unary",le="0.316"} 1 +grpc_server_handling_seconds_bucket{grpc_method="WriteSchema",grpc_service="authzed.api.v1.SchemaService",grpc_type="unary",le="0.562"} 1 +grpc_server_handling_seconds_bucket{grpc_method="WriteSchema",grpc_service="authzed.api.v1.SchemaService",grpc_type="unary",le="1"} 1 +grpc_server_handling_seconds_bucket{grpc_method="WriteSchema",grpc_service="authzed.api.v1.SchemaService",grpc_type="unary",le="5"} 1 +grpc_server_handling_seconds_bucket{grpc_method="WriteSchema",grpc_service="authzed.api.v1.SchemaService",grpc_type="unary",le="+Inf"} 1 +grpc_server_handling_seconds_sum{grpc_method="WriteSchema",grpc_service="authzed.api.v1.SchemaService",grpc_type="unary"} 0.036535555 +grpc_server_handling_seconds_count{grpc_method="WriteSchema",grpc_service="authzed.api.v1.SchemaService",grpc_type="unary"} 1 +# HELP grpc_server_msg_received_total Total number of RPC stream messages received on the server. +# TYPE grpc_server_msg_received_total counter +grpc_server_msg_received_total{grpc_method="ImportBulkRelationships",grpc_service="authzed.api.v1.PermissionsService",grpc_type="client_stream"} 110 +grpc_server_msg_received_total{grpc_method="WriteSchema",grpc_service="authzed.api.v1.SchemaService",grpc_type="unary"} 1 +# HELP grpc_server_msg_sent_total Total number of gRPC stream messages sent by the server. +# TYPE grpc_server_msg_sent_total counter +grpc_server_msg_sent_total{grpc_method="ImportBulkRelationships",grpc_service="authzed.api.v1.PermissionsService",grpc_type="client_stream"} 10 +grpc_server_msg_sent_total{grpc_method="WriteSchema",grpc_service="authzed.api.v1.SchemaService",grpc_type="unary"} 1 +# HELP grpc_server_started_total Total number of RPCs started on the server. +# TYPE grpc_server_started_total counter +grpc_server_started_total{grpc_method="ImportBulkRelationships",grpc_service="authzed.api.v1.PermissionsService",grpc_type="client_stream"} 10 +grpc_server_started_total{grpc_method="WriteSchema",grpc_service="authzed.api.v1.SchemaService",grpc_type="unary"} 1 +# HELP process_cpu_seconds_total Total user and system CPU time spent in seconds. +# TYPE process_cpu_seconds_total counter +process_cpu_seconds_total 0.21 +# HELP process_max_fds Maximum number of open file descriptors. +# TYPE process_max_fds gauge +process_max_fds 524288 +# HELP process_network_receive_bytes_total Number of bytes received by the process over the network. +# TYPE process_network_receive_bytes_total counter +process_network_receive_bytes_total 81639 +# HELP process_network_transmit_bytes_total Number of bytes sent by the process over the network. +# TYPE process_network_transmit_bytes_total counter +process_network_transmit_bytes_total 26309 +# HELP process_open_fds Number of open file descriptors. +# TYPE process_open_fds gauge +process_open_fds 8 +# HELP process_resident_memory_bytes Resident memory size in bytes. +# TYPE process_resident_memory_bytes gauge +process_resident_memory_bytes 4.8422912e+07 +# HELP process_start_time_seconds Start time of the process since unix epoch in seconds. +# TYPE process_start_time_seconds gauge +process_start_time_seconds 1.73170474039e+09 +# HELP process_virtual_memory_bytes Virtual memory size in bytes. +# TYPE process_virtual_memory_bytes gauge +process_virtual_memory_bytes 1.347796992e+09 +# HELP process_virtual_memory_max_bytes Maximum amount of virtual memory available in bytes. +# TYPE process_virtual_memory_max_bytes gauge +process_virtual_memory_max_bytes 1.8446744073709552e+19 +# HELP spicedb_cache_cost_added_bytes Cost of entries added to the cache +# TYPE spicedb_cache_cost_added_bytes counter +spicedb_cache_cost_added_bytes{cache="dispatch"} 0 +spicedb_cache_cost_added_bytes{cache="namespace"} 0 +# HELP spicedb_cache_cost_evicted_bytes Cost of entries evicted from the cache +# TYPE spicedb_cache_cost_evicted_bytes counter +spicedb_cache_cost_evicted_bytes{cache="dispatch"} 0 +spicedb_cache_cost_evicted_bytes{cache="namespace"} 0 +# HELP spicedb_cache_hits_total Number of cache hits +# TYPE spicedb_cache_hits_total counter +spicedb_cache_hits_total{cache="dispatch"} 0 +spicedb_cache_hits_total{cache="namespace"} 0 +# HELP spicedb_cache_misses_total Number of cache misses +# TYPE spicedb_cache_misses_total counter +spicedb_cache_misses_total{cache="dispatch"} 0 +spicedb_cache_misses_total{cache="namespace"} 0 +# HELP spicedb_check_direct_dispatch_query_count number of queries made per direct dispatch +# TYPE spicedb_check_direct_dispatch_query_count histogram +spicedb_check_direct_dispatch_query_count_bucket{le="1"} 0 +spicedb_check_direct_dispatch_query_count_bucket{le="2"} 0 +spicedb_check_direct_dispatch_query_count_bucket{le="+Inf"} 0 +spicedb_check_direct_dispatch_query_count_sum 0 +spicedb_check_direct_dispatch_query_count_count 0 +# HELP spicedb_check_dispatch_chunk_count number of chunks when dispatching in check +# TYPE spicedb_check_dispatch_chunk_count histogram +spicedb_check_dispatch_chunk_count_bucket{le="1"} 0 +spicedb_check_dispatch_chunk_count_bucket{le="2"} 0 +spicedb_check_dispatch_chunk_count_bucket{le="3"} 0 +spicedb_check_dispatch_chunk_count_bucket{le="5"} 0 +spicedb_check_dispatch_chunk_count_bucket{le="10"} 0 +spicedb_check_dispatch_chunk_count_bucket{le="25"} 0 +spicedb_check_dispatch_chunk_count_bucket{le="100"} 0 +spicedb_check_dispatch_chunk_count_bucket{le="250"} 0 +spicedb_check_dispatch_chunk_count_bucket{le="+Inf"} 0 +spicedb_check_dispatch_chunk_count_sum 0 +spicedb_check_dispatch_chunk_count_count 0 +# HELP spicedb_datastore_crdb_watch_retries watch retry distribution +# TYPE spicedb_datastore_crdb_watch_retries histogram +spicedb_datastore_crdb_watch_retries_bucket{le="0"} 0 +spicedb_datastore_crdb_watch_retries_bucket{le="1"} 0 +spicedb_datastore_crdb_watch_retries_bucket{le="2"} 0 +spicedb_datastore_crdb_watch_retries_bucket{le="5"} 0 +spicedb_datastore_crdb_watch_retries_bucket{le="10"} 0 +spicedb_datastore_crdb_watch_retries_bucket{le="20"} 0 +spicedb_datastore_crdb_watch_retries_bucket{le="50"} 0 +spicedb_datastore_crdb_watch_retries_bucket{le="+Inf"} 0 +spicedb_datastore_crdb_watch_retries_sum 0 +spicedb_datastore_crdb_watch_retries_count 0 +# HELP spicedb_datastore_hedgeable_requests_total total number of datastore requests which are eligible for hedging +# TYPE spicedb_datastore_hedgeable_requests_total counter +spicedb_datastore_hedgeable_requests_total 0 +# HELP spicedb_datastore_hedged_requests_total total number of requests which have been hedged +# TYPE spicedb_datastore_hedged_requests_total counter +spicedb_datastore_hedged_requests_total 0 +# HELP spicedb_datastore_loaded_relationships_count total number of relationships loaded for a query +# TYPE spicedb_datastore_loaded_relationships_count histogram +spicedb_datastore_loaded_relationships_count_bucket{le="0"} 0 +spicedb_datastore_loaded_relationships_count_bucket{le="1"} 0 +spicedb_datastore_loaded_relationships_count_bucket{le="3"} 0 +spicedb_datastore_loaded_relationships_count_bucket{le="10"} 0 +spicedb_datastore_loaded_relationships_count_bucket{le="32"} 0 +spicedb_datastore_loaded_relationships_count_bucket{le="100"} 0 +spicedb_datastore_loaded_relationships_count_bucket{le="316"} 0 +spicedb_datastore_loaded_relationships_count_bucket{le="1000"} 0 +spicedb_datastore_loaded_relationships_count_bucket{le="3162"} 0 +spicedb_datastore_loaded_relationships_count_bucket{le="10000"} 0 +spicedb_datastore_loaded_relationships_count_bucket{le="+Inf"} 0 +spicedb_datastore_loaded_relationships_count_sum 0 +spicedb_datastore_loaded_relationships_count_count 0 +# HELP spicedb_datastore_query_latency response latency for a database query +# TYPE spicedb_datastore_query_latency histogram +spicedb_datastore_query_latency_bucket{operation="BulkLoad",le="0.0005"} 8 +spicedb_datastore_query_latency_bucket{operation="BulkLoad",le="0.001"} 8 +spicedb_datastore_query_latency_bucket{operation="BulkLoad",le="0.002"} 12 +spicedb_datastore_query_latency_bucket{operation="BulkLoad",le="0.005"} 19 +spicedb_datastore_query_latency_bucket{operation="BulkLoad",le="0.01"} 20 +spicedb_datastore_query_latency_bucket{operation="BulkLoad",le="0.02"} 20 +spicedb_datastore_query_latency_bucket{operation="BulkLoad",le="0.05"} 20 +spicedb_datastore_query_latency_bucket{operation="BulkLoad",le="0.1"} 20 +spicedb_datastore_query_latency_bucket{operation="BulkLoad",le="0.2"} 20 +spicedb_datastore_query_latency_bucket{operation="BulkLoad",le="0.5"} 20 +spicedb_datastore_query_latency_bucket{operation="BulkLoad",le="+Inf"} 20 +spicedb_datastore_query_latency_sum{operation="BulkLoad"} 0.0360925 +spicedb_datastore_query_latency_count{operation="BulkLoad"} 20 +spicedb_datastore_query_latency_bucket{operation="Features",le="0.0005"} 1 +spicedb_datastore_query_latency_bucket{operation="Features",le="0.001"} 1 +spicedb_datastore_query_latency_bucket{operation="Features",le="0.002"} 1 +spicedb_datastore_query_latency_bucket{operation="Features",le="0.005"} 1 +spicedb_datastore_query_latency_bucket{operation="Features",le="0.01"} 1 +spicedb_datastore_query_latency_bucket{operation="Features",le="0.02"} 1 +spicedb_datastore_query_latency_bucket{operation="Features",le="0.05"} 1 +spicedb_datastore_query_latency_bucket{operation="Features",le="0.1"} 1 +spicedb_datastore_query_latency_bucket{operation="Features",le="0.2"} 1 +spicedb_datastore_query_latency_bucket{operation="Features",le="0.5"} 1 +spicedb_datastore_query_latency_bucket{operation="Features",le="+Inf"} 1 +spicedb_datastore_query_latency_sum{operation="Features"} 7.74e-07 +spicedb_datastore_query_latency_count{operation="Features"} 1 +spicedb_datastore_query_latency_bucket{operation="ListAllCaveats",le="0.0005"} 1 +spicedb_datastore_query_latency_bucket{operation="ListAllCaveats",le="0.001"} 1 +spicedb_datastore_query_latency_bucket{operation="ListAllCaveats",le="0.002"} 1 +spicedb_datastore_query_latency_bucket{operation="ListAllCaveats",le="0.005"} 1 +spicedb_datastore_query_latency_bucket{operation="ListAllCaveats",le="0.01"} 1 +spicedb_datastore_query_latency_bucket{operation="ListAllCaveats",le="0.02"} 1 +spicedb_datastore_query_latency_bucket{operation="ListAllCaveats",le="0.05"} 1 +spicedb_datastore_query_latency_bucket{operation="ListAllCaveats",le="0.1"} 1 +spicedb_datastore_query_latency_bucket{operation="ListAllCaveats",le="0.2"} 1 +spicedb_datastore_query_latency_bucket{operation="ListAllCaveats",le="0.5"} 1 +spicedb_datastore_query_latency_bucket{operation="ListAllCaveats",le="+Inf"} 1 +spicedb_datastore_query_latency_sum{operation="ListAllCaveats"} 0.000403808 +spicedb_datastore_query_latency_count{operation="ListAllCaveats"} 1 +spicedb_datastore_query_latency_bucket{operation="ListAllNamespaces",le="0.0005"} 1 +spicedb_datastore_query_latency_bucket{operation="ListAllNamespaces",le="0.001"} 1 +spicedb_datastore_query_latency_bucket{operation="ListAllNamespaces",le="0.002"} 1 +spicedb_datastore_query_latency_bucket{operation="ListAllNamespaces",le="0.005"} 1 +spicedb_datastore_query_latency_bucket{operation="ListAllNamespaces",le="0.01"} 1 +spicedb_datastore_query_latency_bucket{operation="ListAllNamespaces",le="0.02"} 1 +spicedb_datastore_query_latency_bucket{operation="ListAllNamespaces",le="0.05"} 1 +spicedb_datastore_query_latency_bucket{operation="ListAllNamespaces",le="0.1"} 1 +spicedb_datastore_query_latency_bucket{operation="ListAllNamespaces",le="0.2"} 1 +spicedb_datastore_query_latency_bucket{operation="ListAllNamespaces",le="0.5"} 1 +spicedb_datastore_query_latency_bucket{operation="ListAllNamespaces",le="+Inf"} 1 +spicedb_datastore_query_latency_sum{operation="ListAllNamespaces"} 1.365e-06 +spicedb_datastore_query_latency_count{operation="ListAllNamespaces"} 1 +spicedb_datastore_query_latency_bucket{operation="LookupNamespacesWithNames",le="0.0005"} 10 +spicedb_datastore_query_latency_bucket{operation="LookupNamespacesWithNames",le="0.001"} 10 +spicedb_datastore_query_latency_bucket{operation="LookupNamespacesWithNames",le="0.002"} 10 +spicedb_datastore_query_latency_bucket{operation="LookupNamespacesWithNames",le="0.005"} 10 +spicedb_datastore_query_latency_bucket{operation="LookupNamespacesWithNames",le="0.01"} 10 +spicedb_datastore_query_latency_bucket{operation="LookupNamespacesWithNames",le="0.02"} 10 +spicedb_datastore_query_latency_bucket{operation="LookupNamespacesWithNames",le="0.05"} 10 +spicedb_datastore_query_latency_bucket{operation="LookupNamespacesWithNames",le="0.1"} 10 +spicedb_datastore_query_latency_bucket{operation="LookupNamespacesWithNames",le="0.2"} 10 +spicedb_datastore_query_latency_bucket{operation="LookupNamespacesWithNames",le="0.5"} 10 +spicedb_datastore_query_latency_bucket{operation="LookupNamespacesWithNames",le="+Inf"} 10 +spicedb_datastore_query_latency_sum{operation="LookupNamespacesWithNames"} 0.000252936 +spicedb_datastore_query_latency_count{operation="LookupNamespacesWithNames"} 10 +spicedb_datastore_query_latency_bucket{operation="ReadyState",le="0.0005"} 1 +spicedb_datastore_query_latency_bucket{operation="ReadyState",le="0.001"} 1 +spicedb_datastore_query_latency_bucket{operation="ReadyState",le="0.002"} 1 +spicedb_datastore_query_latency_bucket{operation="ReadyState",le="0.005"} 1 +spicedb_datastore_query_latency_bucket{operation="ReadyState",le="0.01"} 1 +spicedb_datastore_query_latency_bucket{operation="ReadyState",le="0.02"} 1 +spicedb_datastore_query_latency_bucket{operation="ReadyState",le="0.05"} 1 +spicedb_datastore_query_latency_bucket{operation="ReadyState",le="0.1"} 1 +spicedb_datastore_query_latency_bucket{operation="ReadyState",le="0.2"} 1 +spicedb_datastore_query_latency_bucket{operation="ReadyState",le="0.5"} 1 +spicedb_datastore_query_latency_bucket{operation="ReadyState",le="+Inf"} 1 +spicedb_datastore_query_latency_sum{operation="ReadyState"} 6.79e-07 +spicedb_datastore_query_latency_count{operation="ReadyState"} 1 +spicedb_datastore_query_latency_bucket{operation="Statistics",le="0.0005"} 1 +spicedb_datastore_query_latency_bucket{operation="Statistics",le="0.001"} 1 +spicedb_datastore_query_latency_bucket{operation="Statistics",le="0.002"} 1 +spicedb_datastore_query_latency_bucket{operation="Statistics",le="0.005"} 1 +spicedb_datastore_query_latency_bucket{operation="Statistics",le="0.01"} 1 +spicedb_datastore_query_latency_bucket{operation="Statistics",le="0.02"} 1 +spicedb_datastore_query_latency_bucket{operation="Statistics",le="0.05"} 1 +spicedb_datastore_query_latency_bucket{operation="Statistics",le="0.1"} 1 +spicedb_datastore_query_latency_bucket{operation="Statistics",le="0.2"} 1 +spicedb_datastore_query_latency_bucket{operation="Statistics",le="0.5"} 1 +spicedb_datastore_query_latency_bucket{operation="Statistics",le="+Inf"} 1 +spicedb_datastore_query_latency_sum{operation="Statistics"} 8.347e-06 +spicedb_datastore_query_latency_count{operation="Statistics"} 1 +spicedb_datastore_query_latency_bucket{operation="WriteNamespaces",le="0.0005"} 0 +spicedb_datastore_query_latency_bucket{operation="WriteNamespaces",le="0.001"} 0 +spicedb_datastore_query_latency_bucket{operation="WriteNamespaces",le="0.002"} 1 +spicedb_datastore_query_latency_bucket{operation="WriteNamespaces",le="0.005"} 1 +spicedb_datastore_query_latency_bucket{operation="WriteNamespaces",le="0.01"} 1 +spicedb_datastore_query_latency_bucket{operation="WriteNamespaces",le="0.02"} 1 +spicedb_datastore_query_latency_bucket{operation="WriteNamespaces",le="0.05"} 1 +spicedb_datastore_query_latency_bucket{operation="WriteNamespaces",le="0.1"} 1 +spicedb_datastore_query_latency_bucket{operation="WriteNamespaces",le="0.2"} 1 +spicedb_datastore_query_latency_bucket{operation="WriteNamespaces",le="0.5"} 1 +spicedb_datastore_query_latency_bucket{operation="WriteNamespaces",le="+Inf"} 1 +spicedb_datastore_query_latency_sum{operation="WriteNamespaces"} 0.001155418 +spicedb_datastore_query_latency_count{operation="WriteNamespaces"} 1 +# HELP spicedb_datastore_spanner_watch_retries watch retry distribution +# TYPE spicedb_datastore_spanner_watch_retries histogram +spicedb_datastore_spanner_watch_retries_bucket{le="0"} 0 +spicedb_datastore_spanner_watch_retries_bucket{le="1"} 0 +spicedb_datastore_spanner_watch_retries_bucket{le="2"} 0 +spicedb_datastore_spanner_watch_retries_bucket{le="5"} 0 +spicedb_datastore_spanner_watch_retries_bucket{le="10"} 0 +spicedb_datastore_spanner_watch_retries_bucket{le="20"} 0 +spicedb_datastore_spanner_watch_retries_bucket{le="50"} 0 +spicedb_datastore_spanner_watch_retries_bucket{le="+Inf"} 0 +spicedb_datastore_spanner_watch_retries_sum 0 +spicedb_datastore_spanner_watch_retries_count 0 +# HELP spicedb_datastore_watching_schema_cache_caveats_fallback_mode value of 1 if the cache is in fallback mode and 0 otherwise +# TYPE spicedb_datastore_watching_schema_cache_caveats_fallback_mode gauge +spicedb_datastore_watching_schema_cache_caveats_fallback_mode 0 +# HELP spicedb_datastore_watching_schema_cache_namespaces_fallback_mode value of 1 if the cache is in fallback mode and 0 otherwise +# TYPE spicedb_datastore_watching_schema_cache_namespaces_fallback_mode gauge +spicedb_datastore_watching_schema_cache_namespaces_fallback_mode 0 +# HELP spicedb_datastore_watching_schema_cache_tracked_revision the currently tracked max revision for the schema cache +# TYPE spicedb_datastore_watching_schema_cache_tracked_revision gauge +spicedb_datastore_watching_schema_cache_tracked_revision 0 +# HELP spicedb_dispatch_client_check_from_cache_total +# TYPE spicedb_dispatch_client_check_from_cache_total counter +spicedb_dispatch_client_check_from_cache_total 0 +# HELP spicedb_dispatch_client_check_total +# TYPE spicedb_dispatch_client_check_total counter +spicedb_dispatch_client_check_total 0 +# HELP spicedb_dispatch_client_lookup_resources_from_cache_total +# TYPE spicedb_dispatch_client_lookup_resources_from_cache_total counter +spicedb_dispatch_client_lookup_resources_from_cache_total 0 +# HELP spicedb_dispatch_client_lookup_resources_total +# TYPE spicedb_dispatch_client_lookup_resources_total counter +spicedb_dispatch_client_lookup_resources_total 0 +# HELP spicedb_dispatch_client_lookup_subjects_from_cache_total +# TYPE spicedb_dispatch_client_lookup_subjects_from_cache_total counter +spicedb_dispatch_client_lookup_subjects_from_cache_total 0 +# HELP spicedb_dispatch_client_lookup_subjects_total +# TYPE spicedb_dispatch_client_lookup_subjects_total counter +spicedb_dispatch_client_lookup_subjects_total 0 +# HELP spicedb_dispatch_client_reachable_resources_from_cache_total +# TYPE spicedb_dispatch_client_reachable_resources_from_cache_total counter +spicedb_dispatch_client_reachable_resources_from_cache_total 0 +# HELP spicedb_dispatch_client_reachable_resources_total +# TYPE spicedb_dispatch_client_reachable_resources_total counter +spicedb_dispatch_client_reachable_resources_total 0 +# HELP spicedb_services_dispatches Histogram of cluster dispatches performed by the instance. +# TYPE spicedb_services_dispatches histogram +spicedb_services_dispatches_bucket{cached="false",method="ImportBulkRelationships",le="1"} 10 +spicedb_services_dispatches_bucket{cached="false",method="ImportBulkRelationships",le="5"} 10 +spicedb_services_dispatches_bucket{cached="false",method="ImportBulkRelationships",le="10"} 10 +spicedb_services_dispatches_bucket{cached="false",method="ImportBulkRelationships",le="25"} 10 +spicedb_services_dispatches_bucket{cached="false",method="ImportBulkRelationships",le="50"} 10 +spicedb_services_dispatches_bucket{cached="false",method="ImportBulkRelationships",le="100"} 10 +spicedb_services_dispatches_bucket{cached="false",method="ImportBulkRelationships",le="250"} 10 +spicedb_services_dispatches_bucket{cached="false",method="ImportBulkRelationships",le="+Inf"} 10 +spicedb_services_dispatches_sum{cached="false",method="ImportBulkRelationships"} 10 +spicedb_services_dispatches_count{cached="false",method="ImportBulkRelationships"} 10 +spicedb_services_dispatches_bucket{cached="false",method="WriteSchema",le="1"} 0 +spicedb_services_dispatches_bucket{cached="false",method="WriteSchema",le="5"} 1 +spicedb_services_dispatches_bucket{cached="false",method="WriteSchema",le="10"} 1 +spicedb_services_dispatches_bucket{cached="false",method="WriteSchema",le="25"} 1 +spicedb_services_dispatches_bucket{cached="false",method="WriteSchema",le="50"} 1 +spicedb_services_dispatches_bucket{cached="false",method="WriteSchema",le="100"} 1 +spicedb_services_dispatches_bucket{cached="false",method="WriteSchema",le="250"} 1 +spicedb_services_dispatches_bucket{cached="false",method="WriteSchema",le="+Inf"} 1 +spicedb_services_dispatches_sum{cached="false",method="WriteSchema"} 2 +spicedb_services_dispatches_count{cached="false",method="WriteSchema"} 1 +spicedb_services_dispatches_bucket{cached="true",method="ImportBulkRelationships",le="1"} 10 +spicedb_services_dispatches_bucket{cached="true",method="ImportBulkRelationships",le="5"} 10 +spicedb_services_dispatches_bucket{cached="true",method="ImportBulkRelationships",le="10"} 10 +spicedb_services_dispatches_bucket{cached="true",method="ImportBulkRelationships",le="25"} 10 +spicedb_services_dispatches_bucket{cached="true",method="ImportBulkRelationships",le="50"} 10 +spicedb_services_dispatches_bucket{cached="true",method="ImportBulkRelationships",le="100"} 10 +spicedb_services_dispatches_bucket{cached="true",method="ImportBulkRelationships",le="250"} 10 +spicedb_services_dispatches_bucket{cached="true",method="ImportBulkRelationships",le="+Inf"} 10 +spicedb_services_dispatches_sum{cached="true",method="ImportBulkRelationships"} 0 +spicedb_services_dispatches_count{cached="true",method="ImportBulkRelationships"} 10 +spicedb_services_dispatches_bucket{cached="true",method="WriteSchema",le="1"} 1 +spicedb_services_dispatches_bucket{cached="true",method="WriteSchema",le="5"} 1 +spicedb_services_dispatches_bucket{cached="true",method="WriteSchema",le="10"} 1 +spicedb_services_dispatches_bucket{cached="true",method="WriteSchema",le="25"} 1 +spicedb_services_dispatches_bucket{cached="true",method="WriteSchema",le="50"} 1 +spicedb_services_dispatches_bucket{cached="true",method="WriteSchema",le="100"} 1 +spicedb_services_dispatches_bucket{cached="true",method="WriteSchema",le="250"} 1 +spicedb_services_dispatches_bucket{cached="true",method="WriteSchema",le="+Inf"} 1 +spicedb_services_dispatches_sum{cached="true",method="WriteSchema"} 0 +spicedb_services_dispatches_count{cached="true",method="WriteSchema"} 1 diff --git a/observability/simple-datadog/conf.d/openmetrics.d/conf.yaml b/observability/simple-datadog/conf.d/openmetrics.d/conf.yaml new file mode 100644 index 0000000..929a5dc --- /dev/null +++ b/observability/simple-datadog/conf.d/openmetrics.d/conf.yaml @@ -0,0 +1,25 @@ +--- +init_config: + service: "spicedb" + +instances: + - openmetrics_endpoint: "http://spicedb:9090/metrics" + # Prefixes all of the metrics scraped by DD with `spicedb.` + namespace: "spicedb" + metrics: + - grpc_server_handling_seconds: + name: "grpc.server.handling" + type: "histogram" + - grpc_server_handled: + name: "grpc.server.handled" + type: "counter" + # NOTE: for counter metrics that are suffixed by _total, you need to + # remove the suffix for Datadog to pick it up correctly. + - process_cpu_seconds: + name: "process.cpu.seconds" + type: "counter" + - process_virtual_memory_bytes: + name: "process.virtual_memory_bytes" + type: "gauge" + histogram_buckets_as_distributions: true + min_collection_interval: 5 diff --git a/observability/simple-datadog/datadog.yaml b/observability/simple-datadog/datadog.yaml new file mode 100644 index 0000000..be3c67b --- /dev/null +++ b/observability/simple-datadog/datadog.yaml @@ -0,0 +1,5 @@ +--- +# We also don't care about logs +process_config: + container_collection: + enabled: false diff --git a/observability/simple-datadog/docker-compose.yml b/observability/simple-datadog/docker-compose.yml new file mode 100644 index 0000000..bbe16a1 --- /dev/null +++ b/observability/simple-datadog/docker-compose.yml @@ -0,0 +1,90 @@ +--- +name: "metrics-adventure" + +services: + datadog: + image: "datadog/agent" + environment: + DD_API_KEY: "${DD_API_KEY}" + DD_HOSTNAME: "yetibox" + + # NOTE: you can't set this via the config; it needs to come from environment variables. + # These are the environment variables that enable collection via OTLP. + DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_ENDPOINT: "0.0.0.0:4317" + # Tells Datadog that we want to send along traces. + # Additional variables would be needed for OTLP logging and metrics + # if desired. + DD_OTLP_CONFIG_TRACES_ENABLED: true + volumes: + - "./conf.d:/conf.d" + - "./datadog.yaml:/datadog.yaml" + + spicedb: + image: "authzed/spicedb:v1.38.1" + command: "serve" + restart: "on-failure" + environment: + SPICEDB_GRPC_PRESHARED_KEY: "thisisnotasecret" + SPICEDB_DATASTORE_ENGINE: &datastore_engine "postgres" + SPICEDB_DATASTORE_CONN_URI: &datastore_conn_uri "postgres://postgres:secret@database:5432/spicedb?sslmode=disable" + + # OTLP flags; this is how traces are sent to the Datadog agent. + # This matches the receiver endpoint configured on the agent. + SPICEDB_OTEL_ENDPOINT: "datadog:4317" + # Whether to use HTTP or HTTPS + SPICEDB_OTEL_INSECURE: true + # Use gRPC for submission, since the DD agent supports it. `otlphttp` + # is a valid option as well. + SPICEDB_OTEL_PROVIDER: "otlpgrpc" + # Send all traces. This defaults to 0.01 and should be tuned for your system. + SPICEDB_OTEL_SAMPLE_RATIO: 1.0 + # The default is w3c, which datadog doesn't support. In order to get trace + # propagation from a datadog-instrumented application, you'll need to + # ensure that the APM is configured to attach b3 trace propagation header. + SPICEDB_OTEL_TRACE_PROPAGATOR: "b3" + ports: + - "9090:9090" + depends_on: + - "spicedb-migrate" + + spicedb-migrate: + image: "authzed/spicedb:v1.38.1" + command: "migrate head" + restart: "on-failure" + environment: + SPICEDB_DATASTORE_ENGINE: *datastore_engine + SPICEDB_DATASTORE_CONN_URI: *datastore_conn_uri + depends_on: + - "database" + + # Load generation tooling. This is to ensure that there are metrics + # and traces to look at. + thumper: + image: "authzed/thumper" + command: "run --endpoint spicedb:50051 --token thisisnotasecret /scripts/example.yaml" + depends_on: + - "spicedb" + - "thumper-init" + # This runs the schema write for the subsequent load generation. + thumper-init: + image: "authzed/thumper" + command: "migrate --endpoint spicedb:50051 --token thisisnotasecret /scripts/schema.yaml" + # Restarting on failure should mean that the init reattempts until it succeeds + restart: "on-failure" + depends_on: + spicedb: + condition: "service_started" + + database: + image: "postgres:16" + ports: + - "5432:5432" + environment: + - "POSTGRES_PASSWORD=secret" + - "POSTGRES_DB=spicedb" + # This keeps postgres's data around when you bring the system down and back up + volumes: + - "pgdata:/var/lib/postgresql/data" + +volumes: + pgdata: diff --git a/observability/simple-datadog/placeholder.env b/observability/simple-datadog/placeholder.env new file mode 100644 index 0000000..42733cc --- /dev/null +++ b/observability/simple-datadog/placeholder.env @@ -0,0 +1 @@ +DD_API_KEY= diff --git a/observability/simple-datadog/spicedb-dashboard.json b/observability/simple-datadog/spicedb-dashboard.json new file mode 100644 index 0000000..a4d2b73 --- /dev/null +++ b/observability/simple-datadog/spicedb-dashboard.json @@ -0,0 +1 @@ +{"title":"SpiceDB Overview","description":"[[suggested_dashboards]]","widgets":[{"id":512625727012637,"definition":{"title":"New group","banner_img":"","show_title":false,"type":"group","layout_type":"ordered","widgets":[{"id":790129298401198,"definition":{"type":"image","url":"https://design-authzed.vercel.app/images/Banners/SpiceDB-1600x400-Banner-Light.svg","url_dark_theme":"https://design-authzed.vercel.app/images/Banners/SpiceDB-1600x400-Banner.svg","sizing":"contain","has_background":false,"has_border":false,"vertical_align":"center","horizontal_align":"left"},"layout":{"x":0,"y":0,"width":6,"height":2}},{"id":5391176615093781,"definition":{"type":"note","content":"#### Optimize your SpiceDB integration configuration for monitoring\n\n1. Configure `cluster`, `node`, and `region` tags in your `spicedb.d/conf.yaml` file:\n\n\n ```\n instances:\n - openmetrics_endpoint: http://localhost:8080/_status/vars\n tags: [\"cluster:prod\", \"node:1\", \"region:us-east-1\"]\n - openmetrics_endpoint: http://localhost:8082/_status/vars\n tags: [\"cluster:dev\", \"node:1\", \"region:us-east-2\"]\n ```","background_color":"white","font_size":"14","text_align":"left","vertical_align":"center","show_tick":false,"tick_pos":"50%","tick_edge":"left","has_padding":true},"layout":{"x":6,"y":0,"width":6,"height":3}},{"id":8404853868782660,"definition":{"type":"note","content":"This dashboard provides a high-level view of your SpiceDB cluster, including:\n- A high-level view of request performance & latency.\n- Information about resource consumption to help aid in capacity planning.","background_color":"transparent","font_size":"14","text_align":"left","vertical_align":"center","show_tick":false,"tick_pos":"50%","tick_edge":"left","has_padding":false},"layout":{"x":0,"y":2,"width":6,"height":1}}]},"layout":{"x":0,"y":0,"width":12,"height":4}},{"id":3560130955990344,"definition":{"title":"Throughput","show_title":true,"type":"group","layout_type":"ordered","widgets":[{"id":7615141942543467,"definition":{"title":"Total by RPC","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"query1"}],"queries":[{"name":"query1","data_source":"metrics","query":"sum:spicedb.grpc.server.handled.count{*}.as_rate()"}],"response_format":"timeseries","style":{"palette":"dog_classic","order_by":"values","line_type":"solid","line_width":"normal"},"display_type":"bars"}]},"layout":{"x":0,"y":0,"width":6,"height":3}},{"id":3925663272929855,"definition":{"title":"Total by Consistency","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"query1"}],"queries":[{"name":"query1","data_source":"metrics","query":"sum:spicedb.grpc.server.handled.count{*}.as_rate()"}],"response_format":"timeseries","style":{"palette":"dog_classic","order_by":"values","line_type":"solid","line_width":"normal"},"display_type":"bars"}]},"layout":{"x":6,"y":0,"width":6,"height":3}},{"id":364527830066764,"definition":{"title":"Check Operations","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"query1"}],"queries":[{"name":"query1","data_source":"metrics","query":"sum:spicedb.grpc.server.handled.count{grpc_method:checkpermission}.as_rate()"}],"response_format":"timeseries","style":{"palette":"dog_classic","order_by":"values","line_type":"solid","line_width":"normal"},"display_type":"bars"}]},"layout":{"x":0,"y":3,"width":6,"height":3}},{"id":2823083366205463,"definition":{"title":"Lookup Operations","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"query1"}],"queries":[{"name":"query1","data_source":"metrics","query":"sum:spicedb.grpc.server.handled.count{grpc_method in (lookupsubjects ,lookupresources )} by {grpc_method}.as_rate()"}],"response_format":"timeseries","style":{"palette":"dog_classic","order_by":"values","line_type":"solid","line_width":"normal"},"display_type":"bars"}]},"layout":{"x":6,"y":3,"width":6,"height":3}},{"id":279197301579739,"definition":{"title":"Relationship Operations","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"query1"}],"queries":[{"name":"query1","data_source":"metrics","query":"sum:spicedb.grpc.server.handled.count{grpc_method in (readrelationships ,writerelationships ,deleterelationships )} by {grpc_method}.as_rate()"}],"response_format":"timeseries","style":{"palette":"dog_classic","order_by":"values","line_type":"solid","line_width":"normal"},"display_type":"bars"}]},"layout":{"x":0,"y":6,"width":6,"height":3}},{"id":3380816430782344,"definition":{"title":"Schema Operations","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"query1"}],"queries":[{"name":"query1","data_source":"metrics","query":"sum:spicedb.grpc.server.handled.count{grpc_method in (writeschema ,readschema)} by {grpc_method}.as_rate()"}],"response_format":"timeseries","style":{"palette":"dog_classic","order_by":"values","line_type":"solid","line_width":"normal"},"display_type":"bars"}]},"layout":{"x":6,"y":6,"width":6,"height":3}}]},"layout":{"x":0,"y":4,"width":12,"height":10}},{"id":4793433791679627,"definition":{"title":"Latency","show_title":true,"type":"group","layout_type":"ordered","widgets":[{"id":863409587490583,"definition":{"title":"Checks","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"p50: CheckPermission","formula":"query1"},{"alias":"p95: CheckPermission","formula":"query2"},{"alias":"p95: CheckPermission","formula":"query3"}],"queries":[{"name":"query1","data_source":"metrics","query":"p50:spicedb.grpc.server.handling{grpc_method:checkpermission}"},{"name":"query2","data_source":"metrics","query":"p95:spicedb.grpc.server.handling{grpc_method:checkpermission}"},{"name":"query3","data_source":"metrics","query":"p99:spicedb.grpc.server.handling{grpc_method:checkpermission}"}],"response_format":"timeseries","style":{"palette":"dog_classic","order_by":"values","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":0,"y":0,"width":6,"height":3}},{"id":6076165849438225,"definition":{"title":"Bulk Checks","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"p50: CheckBulkPermission","formula":"query1"},{"alias":"p95: CheckBulkPermission","formula":"query2"},{"alias":"p99: CheckBulkPermission","formula":"query3"}],"queries":[{"name":"query1","data_source":"metrics","query":"p50:spicedb.grpc.server.handling{grpc_method in (checkbulkpermission, bulkcheckpermission)}"},{"name":"query2","data_source":"metrics","query":"p95:spicedb.grpc.server.handling{grpc_method in (checkbulkpermission, bulkcheckpermission)}"},{"name":"query3","data_source":"metrics","query":"p99:spicedb.grpc.server.handling{grpc_method in (checkbulkpermission, bulkcheckpermission)}"}],"response_format":"timeseries","style":{"palette":"dog_classic","order_by":"values","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":6,"y":0,"width":6,"height":3}},{"id":5409060165861238,"definition":{"title":"ReadRelationships","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"p50: ReadRelationships","formula":"query1"},{"alias":"p95: WriteRelationships","formula":"query2"},{"alias":"p99: WriteRelationships","formula":"query3"}],"queries":[{"name":"query1","data_source":"metrics","query":"p50:spicedb.grpc.server.handling{grpc_method:readrelationships}"},{"name":"query2","data_source":"metrics","query":"p95:spicedb.grpc.server.handling{grpc_method:readrelationships}"},{"name":"query3","data_source":"metrics","query":"p99:spicedb.grpc.server.handling{grpc_method:readrelationships}"}],"response_format":"timeseries","style":{"palette":"dog_classic","order_by":"values","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":0,"y":3,"width":6,"height":3}},{"id":4680938480636664,"definition":{"title":"WriteRelationships","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"p50: WriteRelationships","formula":"query1"},{"alias":"p95: WriteRelationships","formula":"query2"},{"alias":"p99: WriteRelationships","formula":"query3"}],"queries":[{"name":"query1","data_source":"metrics","query":"p50:spicedb.grpc.server.handling{grpc_method:writerelationships}"},{"name":"query2","data_source":"metrics","query":"p95:spicedb.grpc.server.handling{grpc_method:writerelationships}"},{"name":"query3","data_source":"metrics","query":"p99:spicedb.grpc.server.handling{grpc_method:writerelationships}"}],"response_format":"timeseries","style":{"palette":"dog_classic","order_by":"values","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":6,"y":3,"width":6,"height":3}},{"id":7324769101390434,"definition":{"title":"DeleteRelationships","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"p50: DeleteRelationships","formula":"query1"},{"alias":"p95: DeleteRelationships","formula":"query2"},{"alias":"p99: DeleteRelationships","formula":"query3"}],"queries":[{"name":"query1","data_source":"metrics","query":"p50:spicedb.grpc.server.handling{grpc_method:deleterelationships}"},{"name":"query2","data_source":"metrics","query":"p95:spicedb.grpc.server.handling{grpc_method:deleterelationships}"},{"name":"query3","data_source":"metrics","query":"p99:spicedb.grpc.server.handling{grpc_method:deleterelationships}"}],"response_format":"timeseries","style":{"palette":"dog_classic","order_by":"values","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":0,"y":6,"width":6,"height":3}},{"id":2497504251532424,"definition":{"title":"ExpandPermissionTree","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"p50: ExpandPermissionTree","formula":"query1"},{"alias":"p95: ExpandPermissionTree","formula":"query2"},{"alias":"p99: ExpandPermissionTree","formula":"query3"}],"queries":[{"name":"query1","data_source":"metrics","query":"p50:spicedb.grpc.server.handling{grpc_method:expandpermissiontree}"},{"name":"query2","data_source":"metrics","query":"p95:spicedb.grpc.server.handling{grpc_method:expandpermissiontree}"},{"name":"query3","data_source":"metrics","query":"p99:spicedb.grpc.server.handling{grpc_method:expandpermissiontree}"}],"response_format":"timeseries","style":{"palette":"dog_classic","order_by":"values","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":6,"y":6,"width":6,"height":3}},{"id":8307194191289555,"definition":{"title":"LookupResources","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"p50: LookupResources","formula":"query1"},{"alias":"p95: LookupResources","formula":"query2"},{"alias":"p99: LookupResources","formula":"query3"}],"queries":[{"name":"query1","data_source":"metrics","query":"p50:spicedb.grpc.server.handling{grpc_method:lookupresources}"},{"name":"query2","data_source":"metrics","query":"p95:spicedb.grpc.server.handling{grpc_method:lookupresources}"},{"name":"query3","data_source":"metrics","query":"p99:spicedb.grpc.server.handling{grpc_method:lookupresources}"}],"response_format":"timeseries","style":{"palette":"dog_classic","order_by":"values","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":0,"y":9,"width":6,"height":3}},{"id":8169958283522599,"definition":{"title":"LookupSubjects","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"p50: LookupSubjects","formula":"query1"},{"alias":"p95: LookupSubjects","formula":"query2"},{"alias":"p99: LookupSubjects","formula":"query3"}],"queries":[{"name":"query1","data_source":"metrics","query":"p50:spicedb.grpc.server.handling{grpc_method:lookupsubjects}"},{"name":"query2","data_source":"metrics","query":"p95:spicedb.grpc.server.handling{grpc_method:lookupsubjects}"},{"name":"query3","data_source":"metrics","query":"p99:spicedb.grpc.server.handling{grpc_method:lookupsubjects}"}],"response_format":"timeseries","style":{"palette":"dog_classic","order_by":"values","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":6,"y":9,"width":6,"height":3}},{"id":4835192591489553,"definition":{"title":"ReadSchema","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"p50: ReadSchema","formula":"query1"},{"alias":"p95: ReadSchema","formula":"query2"},{"alias":"p99: ReadSchema","formula":"query3"}],"queries":[{"name":"query1","data_source":"metrics","query":"p50:spicedb.grpc.server.handling{grpc_method:readschema}"},{"name":"query2","data_source":"metrics","query":"p95:spicedb.grpc.server.handling{grpc_method:readschema}"},{"name":"query3","data_source":"metrics","query":"p99:spicedb.grpc.server.handling{grpc_method:readschema}"}],"response_format":"timeseries","style":{"palette":"dog_classic","order_by":"values","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":0,"y":12,"width":6,"height":3}},{"id":1786825540718652,"definition":{"title":"WriteSchema","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"p50: WriteSchema","formula":"query1"},{"alias":"p95: WriteSchema","formula":"query2"},{"alias":"p99: WriteSchema","formula":"query3"}],"queries":[{"name":"query1","data_source":"metrics","query":"p50:spicedb.grpc.server.handling{grpc_method:writeschema}"},{"name":"query2","data_source":"metrics","query":"p95:spicedb.grpc.server.handling{grpc_method:writeschema}"},{"name":"query3","data_source":"metrics","query":"p99:spicedb.grpc.server.handling{grpc_method:writeschema}"}],"response_format":"timeseries","style":{"palette":"dog_classic","order_by":"values","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":6,"y":12,"width":6,"height":3}}]},"layout":{"x":0,"y":14,"width":12,"height":16,"is_column_break":true}},{"id":1969543100199273,"definition":{"title":"Node","show_title":true,"type":"group","layout_type":"ordered","widgets":[{"id":3363586656686623,"definition":{"title":"vCPU","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"CPU %","formula":"query1"}],"queries":[{"name":"query1","data_source":"metrics","query":"sum:spicedb.process.cpu.seconds.count{*} by {host}.as_rate()"}],"response_format":"timeseries","style":{"palette":"dog_classic","order_by":"values","line_type":"solid","line_width":"normal"},"display_type":"line"}],"markers":[]},"layout":{"x":0,"y":0,"width":6,"height":3}},{"id":5592139522821831,"definition":{"title":"Memory","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"Memory Usage","formula":"query1"}],"queries":[{"name":"query1","data_source":"metrics","query":"avg:spicedb.process.virtual_memory_bytes{*} by {host}"}],"response_format":"timeseries","style":{"palette":"dog_classic","order_by":"values","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":6,"y":0,"width":6,"height":3}}]},"layout":{"x":0,"y":30,"width":12,"height":4}}],"template_variables":[],"layout_type":"ordered","notify_list":[],"reflow_type":"fixed"} \ No newline at end of file