diff --git a/.github/workflows/compile-mixin.yml b/.github/workflows/compile-mixin.yml new file mode 100644 index 0000000000..d8950aceb3 --- /dev/null +++ b/.github/workflows/compile-mixin.yml @@ -0,0 +1,119 @@ +name: Compile Pyroscope Mixin + +on: + push: + branches: + - main + paths: + - 'operations/pyroscope/jsonnet/pyroscope-mixin/**' + - 'scripts/compile-mixin.sh' + - '.github/workflows/compile-mixin.yml' + pull_request: + paths: + - 'operations/pyroscope/jsonnet/pyroscope-mixin/**' + - 'scripts/compile-mixin.sh' + - '.github/workflows/compile-mixin.yml' + workflow_dispatch: + +permissions: + contents: read + id-token: write + +jobs: + compile-mixin: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + persist-credentials: 'false' + + - name: Set up Go + uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6 + with: + go-version: 1.23.11 + + - name: Compile mixin + run: make compile-mixin + + - name: Check for changes + id: check-changes + run: | + if ! git diff --exit-code operations/pyroscope/mixin-compiled/; then + echo "changes=true" >> "$GITHUB_OUTPUT" + echo "Compiled mixin files have changed" + else + echo "changes=false" >> "$GITHUB_OUTPUT" + echo "No changes to compiled mixin files" + fi + + - name: Get secrets for PR creation + if: github.event_name == 'push' && github.ref == 'refs/heads/main' && steps.check-changes.outputs.changes == 'true' + id: get-secrets + uses: grafana/shared-workflows/actions/get-vault-secrets@ad819d8e2e2dccb7a28c7e2c43054573d6b45900 + with: + repo_secrets: | + GITHUB_APP_ID=pyroscope-development-app:app-id + GITHUB_APP_INSTALLATION_ID=pyroscope-development-app:app-installation-id + GITHUB_APP_PRIVATE_KEY=pyroscope-development-app:private-key + + - name: Generate token + if: github.event_name == 'push' && github.ref == 'refs/heads/main' && steps.check-changes.outputs.changes == 'true' + id: generate_token + uses: actions/create-github-app-token@67018539274d69449ef7c02e8e71183d1719ab42 # v2.1.4 + with: + app-id: ${{ env.GITHUB_APP_ID }} + private-key: ${{ env.GITHUB_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: | + pyroscope + + - name: Get GitHub App User ID + if: github.event_name == 'push' && github.ref == 'refs/heads/main' && steps.check-changes.outputs.changes == 'true' + id: get-user-id + env: + GITHUB_TOKEN: ${{ steps.generate_token.outputs.token }} + run: | + APP_BOT="${{ steps.generate_token.outputs.app-slug }}[bot]" + echo "user-id=$(gh api "/users/${APP_BOT}" --jq .id)" >> "$GITHUB_OUTPUT" + + - name: Commit and push changes + if: github.event_name == 'push' && github.ref == 'refs/heads/main' && steps.check-changes.outputs.changes == 'true' + env: + GITHUB_TOKEN: ${{ steps.generate_token.outputs.token }} + run: | + APP_BOT="${{ steps.generate_token.outputs.app-slug }}[bot]" + git config --local user.name "${APP_BOT}" + git config --local user.email "${{ steps.get-user-id.outputs.user-id }}+${APP_BOT}@users.noreply.github.com" + git add operations/pyroscope/mixin-compiled/ + git commit -m 'chore: update compiled pyroscope-mixin artifacts + + This commit updates the compiled dashboards and recording rules from the pyroscope-mixin. + + Files updated: + - Dashboards: operations/pyroscope/mixin-compiled/dashboards/ + - Rules: operations/pyroscope/mixin-compiled/rules/ + - Metadata: operations/pyroscope/mixin-compiled/metadata.json + + Generated automatically by compile-mixin.yml workflow.' + gh auth status + git push --force https://x-access-token:${{ steps.generate_token.outputs.token }}@github.com/${{ github.repository }}.git HEAD:main 2> /dev/null + + - name: Check compilation in PR + if: github.event_name == 'pull_request' && steps.check-changes.outputs.changes == 'true' + run: | + echo "::error::Compiled mixin files are out of date. Please run 'make compile-mixin' locally and commit the changes." + exit 1 + + - name: Comment on PR + if: github.event_name == 'pull_request' && steps.check-changes.outputs.changes == 'true' + uses: actions/github-script@v7 + with: + script: | + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: '⚠️ **Compiled mixin artifacts are out of date**\n\nThe pyroscope-mixin source files have changed, but the compiled artifacts in `operations/pyroscope/mixin-compiled/` are not up to date.\n\nPlease run the following command locally and commit the changes:\n```bash\nmake compile-mixin\n```\n\nThen commit and push the updated files:\n```bash\ngit add operations/pyroscope/mixin-compiled/\ngit commit -m "chore: update compiled mixin artifacts"\ngit push\n```' + }) diff --git a/Makefile b/Makefile index 87ee2a6e25..b81a6caa3f 100644 --- a/Makefile +++ b/Makefile @@ -363,6 +363,10 @@ $(BIN)/jb: Makefile go.mod @mkdir -p $(@D) GOBIN=$(abspath $(@D)) $(GO) install github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb@v0.5.1 +$(BIN)/jsonnet: Makefile go.mod + @mkdir -p $(@D) + GOBIN=$(abspath $(@D)) $(GO) install github.com/google/go-jsonnet/cmd/jsonnet@v0.20.0 + $(BIN)/helm: Makefile go.mod @mkdir -p $(@D) CGO_ENABLED=0 GOBIN=$(abspath $(@D)) $(GO) install helm.sh/helm/v3/cmd/helm@v3.14.3 @@ -484,3 +488,11 @@ run: ## Run the pyroscope binary (pass parameters with 'make run PARAMS=-myparam .PHONY: mockery mockery: $(BIN)/mockery $(BIN)/mockery + +.PHONY: compile-mixin +compile-mixin: $(BIN)/jb $(BIN)/jsonnet ## Compile pyroscope-mixin to ready-to-use JSON/YAML files + @PATH="$(BIN):$(PATH)" FROM_MAKEFILE=true ./scripts/compile-mixin.sh + +.PHONY: check/mixin-compiled +check/mixin-compiled: compile-mixin ## Check if mixin-compiled files are up to date + @git --no-pager diff --exit-code operations/pyroscope/mixin-compiled/ || { echo ">> Mixin compiled files are out of date. Run 'make compile-mixin' and commit the changes"; exit 1; } diff --git a/README.md b/README.md index f42384ee23..75dd9bdc56 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,16 @@ For more information on how to use Pyroscope with other programming languages, i * [Deployment Guide](https://grafana.com/docs/pyroscope/latest/deploy-kubernetes/) * [Pyroscope Architecture](https://grafana.com/docs/pyroscope/latest/reference-pyroscope-architecture/) +## Dashboards and Monitoring + +Pre-compiled, ready-to-use Grafana dashboards and Prometheus recording rules are available in [`operations/pyroscope/mixin-compiled/`](operations/pyroscope/mixin-compiled/): + +* **Grafana Dashboards** - Import directly into your Grafana instance to monitor Pyroscope read and write paths +* **Prometheus Recording Rules** - Pre-computed metrics for improved query performance +* **Alert Rules** - Placeholder for custom alerting rules + +[View the dashboards and rules →](operations/pyroscope/mixin-compiled/README.md) + ## Send data to server via Pyroscope agent (language specific) For more documentation on how to add the Pyroscope agent to your code, see the [agent documentation](https://grafana.com/docs/pyroscope/latest/configure-client/) on our website or find language specific examples and documentation below: diff --git a/operations/pyroscope/jsonnet/README.md b/operations/pyroscope/jsonnet/README.md index 64eece94ff..dd7475fbe2 100644 --- a/operations/pyroscope/jsonnet/README.md +++ b/operations/pyroscope/jsonnet/README.md @@ -2,3 +2,37 @@ This folder contains the Jsonnet for deploying Grafana Pyroscope in Kubernetes. The documentation for the Pyroscope Jsonnet is published at [https://grafana.com/docs/pyroscope/latest/deploy-kubernetes/tanka-jsonnet/](https://grafana.com/docs/pyroscope/latest/deploy-kubernetes/tanka-jsonnet/). + +## Pre-compiled Dashboards and Rules + +Ready-to-use Grafana dashboards and Prometheus recording rules compiled from the `pyroscope-mixin/` are available in [`../mixin-compiled/`](../mixin-compiled/): + +- **Grafana Dashboards** - JSON files ready to import into Grafana + - `pyroscope-reads.json` - Read path monitoring + - `pyroscope-writes.json` - Write path monitoring +- **Prometheus Rules** - YAML files ready to load into Prometheus + - `recording-rules.yaml` - Pre-computed metrics for performance + - `alert-rules.yaml` - Placeholder for custom alerts + +### Quick Usage + +```bash +# Import dashboards into Grafana +cd ../mixin-compiled/dashboards/ +# Use Grafana UI to import *.json files + +# Load rules into Prometheus +cd ../mixin-compiled/rules/ +# Add to your prometheus.yml rule_files section +``` + +### Recompiling + +If you modify the mixin source files, recompile with: + +```bash +cd ../../.. # Back to repo root +make compile-mixin +``` + +Files are automatically kept in sync via CI when mixin source files change. diff --git a/operations/pyroscope/mixin-compiled/dashboards/pyroscope-reads.json b/operations/pyroscope/mixin-compiled/dashboards/pyroscope-reads.json new file mode 100644 index 0000000000..0635851444 --- /dev/null +++ b/operations/pyroscope/mixin-compiled/dashboards/pyroscope-reads.json @@ -0,0 +1,504 @@ +{ + "annotations": { + "list": [ ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "pyroscope" + ], + "targetBlank": false, + "title": "Pyroscope Dashboards", + "type": "dashboards" + } + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status) (\n label_replace(label_replace(rate(pyroscope_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(pyroscope|querier)\", route=~\".*merge.*|.*series.*|.*type.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{status}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "QPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum by (le,route) (job_route_cluster:pyroscope_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(pyroscope|querier)\", route=~\".*merge.*|.*series.*|.*type.*\", cluster=~\"$cluster\"})) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ route }} 99th Percentile", + "refId": "A", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum by (le,route) (job_route_cluster:pyroscope_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(pyroscope|querier)\", route=~\".*merge.*|.*series.*|.*type.*\", cluster=~\"$cluster\"})) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ route }} 50th Percentile", + "refId": "B", + "step": 10 + }, + { + "expr": "1e3 * sum(job_route_cluster:pyroscope_request_duration_seconds_sum:sum_rate{job=~\"($namespace)/(pyroscope|querier)\", route=~\".*merge.*|.*series.*|.*type.*\", cluster=~\"$cluster\"}) by (route) / sum(job_route_cluster:pyroscope_request_duration_seconds_count:sum_rate{job=~\"($namespace)/(pyroscope|querier)\", route=~\".*merge.*|.*series.*|.*type.*\", cluster=~\"$cluster\"}) by (route) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ route }} Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Querier", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status) (\n label_replace(label_replace(rate(pyroscope_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(pyroscope|ingester)\", route=~\".*merge.*|.*series.*|.*type.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{status}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "QPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum by (le,route) (job_route_cluster:pyroscope_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(pyroscope|ingester)\", route=~\".*merge.*|.*series.*|.*type.*\", cluster=~\"$cluster\"})) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ route }} 99th Percentile", + "refId": "A", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum by (le,route) (job_route_cluster:pyroscope_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(pyroscope|ingester)\", route=~\".*merge.*|.*series.*|.*type.*\", cluster=~\"$cluster\"})) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ route }} 50th Percentile", + "refId": "B", + "step": 10 + }, + { + "expr": "1e3 * sum(job_route_cluster:pyroscope_request_duration_seconds_sum:sum_rate{job=~\"($namespace)/(pyroscope|ingester)\", route=~\".*merge.*|.*series.*|.*type.*\", cluster=~\"$cluster\"}) by (route) / sum(job_route_cluster:pyroscope_request_duration_seconds_count:sum_rate{job=~\"($namespace)/(pyroscope|ingester)\", route=~\".*merge.*|.*series.*|.*type.*\", cluster=~\"$cluster\"}) by (route) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ route }} Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Ingester", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "pyroscope" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ ], + "query": "label_values(pyroscope_build_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ ], + "query": "label_values(pyroscope_build_info{cluster=~\"$cluster\"}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Pyroscope / Reads", + "uid": "pyroscope-reads", + "version": 0 +} diff --git a/operations/pyroscope/mixin-compiled/dashboards/pyroscope-writes.json b/operations/pyroscope/mixin-compiled/dashboards/pyroscope-writes.json new file mode 100644 index 0000000000..e65930626e --- /dev/null +++ b/operations/pyroscope/mixin-compiled/dashboards/pyroscope-writes.json @@ -0,0 +1,866 @@ +{ + "annotations": { + "list": [ ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "pyroscope" + ], + "targetBlank": false, + "title": "Pyroscope Dashboards", + "type": "dashboards" + } + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum by (le,type) (job_type_cluster:pyroscope_distributor_received_compressed_bytes_bucket:sum_rate{job=~\"($namespace)/(pyroscope|distributor)\", type=~\".*\", cluster=~\"$cluster\"})) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ type }} 99th Percentile", + "refId": "A", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum by (le,type) (job_type_cluster:pyroscope_distributor_received_compressed_bytes_bucket:sum_rate{job=~\"($namespace)/(pyroscope|distributor)\", type=~\".*\", cluster=~\"$cluster\"})) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ type }} 50th Percentile", + "refId": "B", + "step": 10 + }, + { + "expr": "1 * sum(job_type_cluster:pyroscope_distributor_received_compressed_bytes_sum:sum_rate{job=~\"($namespace)/(pyroscope|distributor)\", type=~\".*\", cluster=~\"$cluster\"}) by (type) / sum(job_type_cluster:pyroscope_distributor_received_compressed_bytes_count:sum_rate{job=~\"($namespace)/(pyroscope|distributor)\", type=~\".*\", cluster=~\"$cluster\"}) by (type) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ type }} Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Compressed Size", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum by (le,type) (job_type_cluster:pyroscope_distributor_received_samples_bucket:sum_rate{job=~\"($namespace)/(pyroscope|distributor)\", type=~\".*\", cluster=~\"$cluster\"})) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ type }} 99th Percentile", + "refId": "A", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum by (le,type) (job_type_cluster:pyroscope_distributor_received_samples_bucket:sum_rate{job=~\"($namespace)/(pyroscope|distributor)\", type=~\".*\", cluster=~\"$cluster\"})) * 1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ type }} 50th Percentile", + "refId": "B", + "step": 10 + }, + { + "expr": "1 * sum(job_type_cluster:pyroscope_distributor_received_samples_sum:sum_rate{job=~\"($namespace)/(pyroscope|distributor)\", type=~\".*\", cluster=~\"$cluster\"}) by (type) / sum(job_type_cluster:pyroscope_distributor_received_samples_count:sum_rate{job=~\"($namespace)/(pyroscope|distributor)\", type=~\".*\", cluster=~\"$cluster\"}) by (type) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ type }} Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Samples", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "count", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Distributor Profiles received", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status) (\n label_replace(label_replace(rate(pyroscope_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(pyroscope|distributor)\", route=~\".*push.*|.*ingest.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{status}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "QPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum by (le) (job_route_cluster:pyroscope_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(pyroscope|distributor)\", route=~\".*push.*\", cluster=~\"$cluster\"})) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99th Percentile", + "refId": "A", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum by (le) (job_route_cluster:pyroscope_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(pyroscope|distributor)\", route=~\".*push.*\", cluster=~\"$cluster\"})) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "50th Percentile", + "refId": "B", + "step": 10 + }, + { + "expr": "1e3 * sum(job_route_cluster:pyroscope_request_duration_seconds_sum:sum_rate{job=~\"($namespace)/(pyroscope|distributor)\", route=~\".*push.*\", cluster=~\"$cluster\"}) / sum(job_route_cluster:pyroscope_request_duration_seconds_count:sum_rate{job=~\"($namespace)/(pyroscope|distributor)\", route=~\".*push.*\", cluster=~\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Distributor Requests", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status) (\n label_replace(label_replace(rate(pyroscope_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(pyroscope|ingester)\", route=~\".*push.*|.*ingest.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{status}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "QPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum by (le) (job_route_cluster:pyroscope_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(pyroscope|ingester)\", route=~\".*push.*|.*ingest.*\", cluster=~\"$cluster\"})) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99th Percentile", + "refId": "A", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum by (le) (job_route_cluster:pyroscope_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(pyroscope|ingester)\", route=~\".*push.*|.*ingest.*\", cluster=~\"$cluster\"})) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "50th Percentile", + "refId": "B", + "step": 10 + }, + { + "expr": "1e3 * sum(job_route_cluster:pyroscope_request_duration_seconds_sum:sum_rate{job=~\"($namespace)/(pyroscope|ingester)\", route=~\".*push.*|.*ingest.*\", cluster=~\"$cluster\"}) / sum(job_route_cluster:pyroscope_request_duration_seconds_count:sum_rate{job=~\"($namespace)/(pyroscope|ingester)\", route=~\".*push.*|.*ingest.*\", cluster=~\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Latency", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Ingester", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "### Head size in bytes per table type\nIngesters maintain a local Head per-tenant. Each\nHead maintains the active profiling series; Then\nthe head gets periodically compacted into a block\non disk. This panel shows the estimated size of\nthe Head in memory for all ingesters.\n\n", + "fill": 1, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(pyroscope_head_size_bytes{cluster=~\"$cluster\",job=~\"($namespace)/(pyroscope|ingester)\",}) by (type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Head size in bytes per table type", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "### Head size in bytes per pod\nIngesters maintain a local Head per-tenant. Each\nHead maintains the active profiling series; Then\nthe head gets periodically compacted into a block\non disk. This panel shows the estimated size of\nthe Head in memory for all ingesters.\n\n", + "fill": 1, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(pyroscope_head_size_bytes{cluster=~\"$cluster\",job=~\"($namespace)/(pyroscope|ingester)\",}) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Head size in bytes per pod", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Ingester - Head", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "pyroscope" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ ], + "query": "label_values(pyroscope_build_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ ], + "query": "label_values(pyroscope_build_info{cluster=~\"$cluster\"}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Pyroscope / Writes", + "uid": "pyroscope-writes", + "version": 0 +} diff --git a/operations/pyroscope/mixin-compiled/metadata.json b/operations/pyroscope/mixin-compiled/metadata.json new file mode 100644 index 0000000000..7efb7ea393 --- /dev/null +++ b/operations/pyroscope/mixin-compiled/metadata.json @@ -0,0 +1,10 @@ +{ + "compiled_at": "2025-10-15T17:58:40Z", + "git_commit": "a96bbfd58", + "git_branch": "main", + "source": "operations/pyroscope/jsonnet/pyroscope-mixin/pyroscope-mixin", + "compiler_version": { + "jsonnet": "Jsonnet commandline interpreter (Go implementation) v0.20.0", + "jsonnet-bundler": "dev" + } +} diff --git a/operations/pyroscope/mixin-compiled/rules/alert-rules.yaml b/operations/pyroscope/mixin-compiled/rules/alert-rules.yaml new file mode 100644 index 0000000000..802598d900 --- /dev/null +++ b/operations/pyroscope/mixin-compiled/rules/alert-rules.yaml @@ -0,0 +1,9 @@ +# Pyroscope Alert Rules +# +# Currently, no alert rules are defined in the pyroscope-mixin. +# This file is a placeholder for future alert rules. +# +# To add your own alert rules, you can create them here following +# the Prometheus alerting rules format: +# https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/ +groups: [] diff --git a/operations/pyroscope/mixin-compiled/rules/recording-rules.yaml b/operations/pyroscope/mixin-compiled/rules/recording-rules.yaml new file mode 100644 index 0000000000..9f9319b793 --- /dev/null +++ b/operations/pyroscope/mixin-compiled/rules/recording-rules.yaml @@ -0,0 +1,129 @@ +{ + "groups": [ + { + "name": "pyroscope_rules", + "rules": [ + { + "expr": "histogram_quantile(0.99, sum(rate(pyroscope_request_duration_seconds_bucket[1m])) by (le, job, cluster))", + "record": "job_cluster:pyroscope_request_duration_seconds:99quantile" + }, + { + "expr": "histogram_quantile(0.50, sum(rate(pyroscope_request_duration_seconds_bucket[1m])) by (le, job, cluster))", + "record": "job_cluster:pyroscope_request_duration_seconds:50quantile" + }, + { + "expr": "sum(rate(pyroscope_request_duration_seconds_sum[1m])) by (job, cluster) / sum(rate(pyroscope_request_duration_seconds_count[1m])) by (job, cluster)", + "record": "job_cluster:pyroscope_request_duration_seconds:avg" + }, + { + "expr": "sum(rate(pyroscope_request_duration_seconds_bucket[1m])) by (le, job, cluster)", + "record": "job_cluster:pyroscope_request_duration_seconds_bucket:sum_rate" + }, + { + "expr": "sum(rate(pyroscope_request_duration_seconds_sum[1m])) by (job, cluster)", + "record": "job_cluster:pyroscope_request_duration_seconds_sum:sum_rate" + }, + { + "expr": "sum(rate(pyroscope_request_duration_seconds_count[1m])) by (job, cluster)", + "record": "job_cluster:pyroscope_request_duration_seconds_count:sum_rate" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(pyroscope_request_duration_seconds_bucket[1m])) by (le, job, route, cluster))", + "record": "job_route_cluster:pyroscope_request_duration_seconds:99quantile" + }, + { + "expr": "histogram_quantile(0.50, sum(rate(pyroscope_request_duration_seconds_bucket[1m])) by (le, job, route, cluster))", + "record": "job_route_cluster:pyroscope_request_duration_seconds:50quantile" + }, + { + "expr": "sum(rate(pyroscope_request_duration_seconds_sum[1m])) by (job, route, cluster) / sum(rate(pyroscope_request_duration_seconds_count[1m])) by (job, route, cluster)", + "record": "job_route_cluster:pyroscope_request_duration_seconds:avg" + }, + { + "expr": "sum(rate(pyroscope_request_duration_seconds_bucket[1m])) by (le, job, route, cluster)", + "record": "job_route_cluster:pyroscope_request_duration_seconds_bucket:sum_rate" + }, + { + "expr": "sum(rate(pyroscope_request_duration_seconds_sum[1m])) by (job, route, cluster)", + "record": "job_route_cluster:pyroscope_request_duration_seconds_sum:sum_rate" + }, + { + "expr": "sum(rate(pyroscope_request_duration_seconds_count[1m])) by (job, route, cluster)", + "record": "job_route_cluster:pyroscope_request_duration_seconds_count:sum_rate" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(pyroscope_request_duration_seconds_bucket[1m])) by (le, namespace, job, route, cluster))", + "record": "namespace_job_route_cluster:pyroscope_request_duration_seconds:99quantile" + }, + { + "expr": "histogram_quantile(0.50, sum(rate(pyroscope_request_duration_seconds_bucket[1m])) by (le, namespace, job, route, cluster))", + "record": "namespace_job_route_cluster:pyroscope_request_duration_seconds:50quantile" + }, + { + "expr": "sum(rate(pyroscope_request_duration_seconds_sum[1m])) by (namespace, job, route, cluster) / sum(rate(pyroscope_request_duration_seconds_count[1m])) by (namespace, job, route, cluster)", + "record": "namespace_job_route_cluster:pyroscope_request_duration_seconds:avg" + }, + { + "expr": "sum(rate(pyroscope_request_duration_seconds_bucket[1m])) by (le, namespace, job, route, cluster)", + "record": "namespace_job_route_cluster:pyroscope_request_duration_seconds_bucket:sum_rate" + }, + { + "expr": "sum(rate(pyroscope_request_duration_seconds_sum[1m])) by (namespace, job, route, cluster)", + "record": "namespace_job_route_cluster:pyroscope_request_duration_seconds_sum:sum_rate" + }, + { + "expr": "sum(rate(pyroscope_request_duration_seconds_count[1m])) by (namespace, job, route, cluster)", + "record": "namespace_job_route_cluster:pyroscope_request_duration_seconds_count:sum_rate" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(pyroscope_distributor_received_compressed_bytes_bucket[1m])) by (le, job, type, cluster))", + "record": "job_type_cluster:pyroscope_distributor_received_compressed_bytes:99quantile" + }, + { + "expr": "histogram_quantile(0.50, sum(rate(pyroscope_distributor_received_compressed_bytes_bucket[1m])) by (le, job, type, cluster))", + "record": "job_type_cluster:pyroscope_distributor_received_compressed_bytes:50quantile" + }, + { + "expr": "sum(rate(pyroscope_distributor_received_compressed_bytes_sum[1m])) by (job, type, cluster) / sum(rate(pyroscope_distributor_received_compressed_bytes_count[1m])) by (job, type, cluster)", + "record": "job_type_cluster:pyroscope_distributor_received_compressed_bytes:avg" + }, + { + "expr": "sum(rate(pyroscope_distributor_received_compressed_bytes_bucket[1m])) by (le, job, type, cluster)", + "record": "job_type_cluster:pyroscope_distributor_received_compressed_bytes_bucket:sum_rate" + }, + { + "expr": "sum(rate(pyroscope_distributor_received_compressed_bytes_sum[1m])) by (job, type, cluster)", + "record": "job_type_cluster:pyroscope_distributor_received_compressed_bytes_sum:sum_rate" + }, + { + "expr": "sum(rate(pyroscope_distributor_received_compressed_bytes_count[1m])) by (job, type, cluster)", + "record": "job_type_cluster:pyroscope_distributor_received_compressed_bytes_count:sum_rate" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(pyroscope_distributor_received_samples_bucket[1m])) by (le, job, type, cluster))", + "record": "job_type_cluster:pyroscope_distributor_received_samples:99quantile" + }, + { + "expr": "histogram_quantile(0.50, sum(rate(pyroscope_distributor_received_samples_bucket[1m])) by (le, job, type, cluster))", + "record": "job_type_cluster:pyroscope_distributor_received_samples:50quantile" + }, + { + "expr": "sum(rate(pyroscope_distributor_received_samples_sum[1m])) by (job, type, cluster) / sum(rate(pyroscope_distributor_received_samples_count[1m])) by (job, type, cluster)", + "record": "job_type_cluster:pyroscope_distributor_received_samples:avg" + }, + { + "expr": "sum(rate(pyroscope_distributor_received_samples_bucket[1m])) by (le, job, type, cluster)", + "record": "job_type_cluster:pyroscope_distributor_received_samples_bucket:sum_rate" + }, + { + "expr": "sum(rate(pyroscope_distributor_received_samples_sum[1m])) by (job, type, cluster)", + "record": "job_type_cluster:pyroscope_distributor_received_samples_sum:sum_rate" + }, + { + "expr": "sum(rate(pyroscope_distributor_received_samples_count[1m])) by (job, type, cluster)", + "record": "job_type_cluster:pyroscope_distributor_received_samples_count:sum_rate" + } + ] + } + ] +} diff --git a/scripts/compile-mixin.sh b/scripts/compile-mixin.sh new file mode 100755 index 0000000000..5c95d18369 --- /dev/null +++ b/scripts/compile-mixin.sh @@ -0,0 +1,233 @@ +#!/usr/bin/env bash + +# Script to compile Pyroscope jsonnet mixin to JSON/YAML files +# This script compiles dashboards, recording rules, and alert rules +# from the pyroscope-mixin into ready-to-use files. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +MIXIN_DIR="${PROJECT_ROOT}/operations/pyroscope/jsonnet/pyroscope-mixin/pyroscope-mixin" +OUTPUT_DIR="${PROJECT_ROOT}/operations/pyroscope/mixin-compiled" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +log_info() { + echo -e "${GREEN}[INFO]${NC} $1" +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Check if required tools are available +check_dependencies() { + local missing_deps=() + + if ! command -v jb &> /dev/null; then + missing_deps+=("jb (jsonnet-bundler)") + fi + + if ! command -v jsonnet &> /dev/null; then + missing_deps+=("jsonnet") + fi + + if [ ${#missing_deps[@]} -ne 0 ]; then + log_error "Missing required dependencies: ${missing_deps[*]}" + log_info "Please install them or run: make compile-mixin (which will install them automatically)" + exit 1 + fi +} + +# Install jsonnet dependencies +install_dependencies() { + log_info "Installing jsonnet dependencies..." + cd "${MIXIN_DIR}" + + if [ -f "jsonnetfile.json" ]; then + jb install + log_info "Dependencies installed successfully" + else + log_warn "No jsonnetfile.json found, skipping dependency installation" + fi +} + +# Create output directories +setup_output_dirs() { + log_info "Setting up output directories..." + mkdir -p "${OUTPUT_DIR}/dashboards" + mkdir -p "${OUTPUT_DIR}/rules" +} + +# Compile dashboards +compile_dashboards() { + log_info "Compiling dashboards..." + + cd "${MIXIN_DIR}" + + # Create a temporary jsonnet file to extract dashboards + cat > "${MIXIN_DIR}/compile-dashboards.jsonnet" <<'EOF' +local mixin = import 'mixin.libsonnet'; +{ + [name]: mixin.grafanaDashboards[name] + for name in std.objectFields(mixin.grafanaDashboards) +} +EOF + + # Compile all dashboards + jsonnet -J vendor compile-dashboards.jsonnet -m "${OUTPUT_DIR}/dashboards" + + # Clean up + rm -f "${MIXIN_DIR}/compile-dashboards.jsonnet" + + # Count and report + local dashboard_count=$(find "${OUTPUT_DIR}/dashboards" -name "*.json" | wc -l | tr -d ' ') + log_info "Compiled ${dashboard_count} dashboard(s)" +} + +# Compile recording rules +compile_recording_rules() { + log_info "Compiling recording rules..." + + cd "${MIXIN_DIR}" + + # Create a temporary jsonnet file to extract recording rules + cat > "${MIXIN_DIR}/compile-recording-rules.jsonnet" <<'EOF' +local mixin = import 'mixin.libsonnet'; +mixin.prometheusRules +EOF + + # Compile recording rules to YAML (Prometheus format) + jsonnet -J vendor compile-recording-rules.jsonnet -o "${OUTPUT_DIR}/rules/recording-rules.json" + + # Convert JSON to YAML using Python (more reliable than jsonnet -y) + python3 -c "import json, yaml, sys; print(yaml.dump(json.load(open('${OUTPUT_DIR}/rules/recording-rules.json')), default_flow_style=False))" > "${OUTPUT_DIR}/rules/recording-rules.yaml" 2>/dev/null || { + # Fallback: keep JSON if yaml module not available + log_warn "Python yaml module not found, keeping JSON format" + mv "${OUTPUT_DIR}/rules/recording-rules.json" "${OUTPUT_DIR}/rules/recording-rules.yaml" + } + + # Clean up JSON file if YAML was created successfully + [ -f "${OUTPUT_DIR}/rules/recording-rules.yaml" ] && rm -f "${OUTPUT_DIR}/rules/recording-rules.json" + + # Clean up + rm -f "${MIXIN_DIR}/compile-recording-rules.jsonnet" + + log_info "Compiled recording rules to recording-rules.yaml" +} + +# Compile alert rules (if they exist) +compile_alert_rules() { + log_info "Checking for alert rules..." + + cd "${MIXIN_DIR}" + + # Create a temporary jsonnet file to check for alerts + cat > "${MIXIN_DIR}/compile-alert-rules.jsonnet" <<'EOF' +local mixin = import 'mixin.libsonnet'; +if std.objectHas(mixin, 'prometheusAlerts') then + mixin.prometheusAlerts +else + {} +EOF + + # Check if alerts exist + local alert_output=$(jsonnet -J vendor compile-alert-rules.jsonnet 2>/dev/null || echo "{}") + + if [ "${alert_output}" != "{}" ] && [ "${alert_output}" != "{ }" ]; then + jsonnet -J vendor compile-alert-rules.jsonnet -o "${OUTPUT_DIR}/rules/alert-rules.json" + + # Convert JSON to YAML using Python + python3 -c "import json, yaml, sys; print(yaml.dump(json.load(open('${OUTPUT_DIR}/rules/alert-rules.json')), default_flow_style=False))" > "${OUTPUT_DIR}/rules/alert-rules.yaml" 2>/dev/null || { + log_warn "Python yaml module not found, keeping JSON format" + mv "${OUTPUT_DIR}/rules/alert-rules.json" "${OUTPUT_DIR}/rules/alert-rules.yaml" + } + + # Clean up JSON file + [ -f "${OUTPUT_DIR}/rules/alert-rules.yaml" ] && rm -f "${OUTPUT_DIR}/rules/alert-rules.json" + + log_info "Compiled alert rules to alert-rules.yaml" + else + log_warn "No alert rules found in mixin" + # Create an empty placeholder file with documentation + cat > "${OUTPUT_DIR}/rules/alert-rules.yaml" <<'YAML_EOF' +# Pyroscope Alert Rules +# +# Currently, no alert rules are defined in the pyroscope-mixin. +# This file is a placeholder for future alert rules. +# +# To add your own alert rules, you can create them here following +# the Prometheus alerting rules format: +# https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/ +groups: [] +YAML_EOF + log_info "Created placeholder alert-rules.yaml" + fi + + # Clean up + rm -f "${MIXIN_DIR}/compile-alert-rules.jsonnet" +} + +# Generate metadata file +generate_metadata() { + log_info "Generating metadata..." + + local git_commit=$(git -C "${PROJECT_ROOT}" rev-parse --short HEAD 2>/dev/null || echo "unknown") + local git_branch=$(git -C "${PROJECT_ROOT}" rev-parse --abbrev-ref HEAD 2>/dev/null || echo "unknown") + local compile_date=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + + cat > "${OUTPUT_DIR}/metadata.json" <&1 | head -n 1 || echo 'unknown')", + "jsonnet-bundler": "$(jb --version 2>&1 || echo 'unknown')" + } +} +EOF + + log_info "Generated metadata.json" +} + +# Main execution +main() { + log_info "Starting Pyroscope mixin compilation..." + log_info "Mixin source: ${MIXIN_DIR}" + log_info "Output directory: ${OUTPUT_DIR}" + + # Check if running from Makefile (dependencies already installed) + if [ "${FROM_MAKEFILE:-}" != "true" ]; then + check_dependencies + fi + + install_dependencies + setup_output_dirs + compile_dashboards + compile_recording_rules + compile_alert_rules + generate_metadata + + log_info "" + log_info "✓ Compilation complete!" + log_info "Output location: ${OUTPUT_DIR}" + log_info "" + log_info "Files generated:" + log_info " - Dashboards: ${OUTPUT_DIR}/dashboards/" + log_info " - Recording Rules: ${OUTPUT_DIR}/rules/recording-rules.yaml" + log_info " - Alert Rules: ${OUTPUT_DIR}/rules/alert-rules.yaml" + log_info " - Metadata: ${OUTPUT_DIR}/metadata.json" +} + +main "$@"