feat: Helm support for v1/v2 storage (#4417)

* WIP: Helm support for v1/v2

* Make helm deployable again

* Milestone: v2 micros-services works

* Fix folders and search addressed to fix single-binary v2

* Update helm-docs to support ignored

Also install it using go get instead docker

* Build helm always without CGO

* Introduce cluster label change when switching from single binary to
micro-services

Without that there is a chance the single binary instances are kept in
the ring without being cleaned up.

* Provide info about the migration in the NOTES

* Add some WIP v2 migration docs for helm

* Update migration guide

* Upgrade helm-ci

* Update image for super linter

* Fix linting issues

* Call helm-docs correctly

* Fix lint check

* Make superlinter work

* Not too sure super linter brings any value

* Apply suggestions from code review

Co-authored-by: Alberto <[email protected]>

* Apply suggestions from review everywhere

---------

Co-authored-by: Alberto <[email protected]>

Author: simonswine

.github/workflows/helm-ci.yml

@@ -3,10 +3,21 @@ on: pull_request
 permissions:
   contents: read
 jobs:
-  call-lint:
-    uses: grafana/helm-charts/.github/workflows/linter.yml@main
-    with:
-      filter_regex_include: operations/pyroscope/helm/pyroscope/templates/.*
+  # We used to import this from grafana/helm-charts/.github/workflows/linter.yml@main, but we want to update to later version of helm-docs.
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+        with:
+          persist-credentials: false
+      - name: Check Docs
+        run: |
+          make helm/docs
+          if ! git diff --exit-code -- operations/pyroscope/helm/pyroscope/; then
+              echo 'Documentation not up to date. Please run "make helm/docs" and commit changes!' >&2
+              exit 1
+          fi
 
   call-lint-test:
     uses: grafana/helm-charts/.github/workflows/lint-test.yaml@main

Makefile

@@ -46,6 +46,12 @@ GO_MOD_PATHS := api/ lidia/ examples/language-sdk-instrumentation/golang-push/ri
 # Add extra arguments to helm commands
 HELM_ARGS =
 
+HELM_FLAGS_V1 :=
+HELM_FLAGS_V1_MICROSERVICES := --set architecture.microservices.enabled=true --set minio.enabled=true
+HELM_FLAGS_V2 := --set architecture.storage.v1=false --set architecture.storage.v2=true
+HELM_FLAGS_V2_MICROSERVICES := $(HELM_FLAGS_V1_MICROSERVICES) $(HELM_FLAGS_V2)
+
+
 # Local deployment params
 KIND_CLUSTER = pyroscope-dev
 
@@ -223,7 +229,9 @@ define deploy
 	# Load image into nodes
 	$(BIN)/kind load docker-image --name $(KIND_CLUSTER) $(IMAGE_PREFIX)pyroscope:$(IMAGE_TAG)
 	kubectl get pods
-	$(BIN)/helm upgrade --install $(1) ./operations/pyroscope/helm/pyroscope $(2) $(HELM_ARGS) \
+	$(BIN)/helm upgrade --install pyroscope ./operations/pyroscope/helm/pyroscope $(2) $(HELM_ARGS) \
+		--set architecture.deployUnifiedServices=true \
+		--set architecture.overwriteResources.requests.cpu=10m \
 		--set pyroscope.image.tag=$(IMAGE_TAG) \
 		--set pyroscope.image.repository=$(IMAGE_PREFIX)pyroscope \
 		--set pyroscope.podAnnotations.image-digest=$(shell cat .docker-image-digest-pyroscope) \
@@ -280,7 +288,8 @@ docker-image/pyroscope/push-debug: frontend/build go/bin-debug docker-image/pyro
 	$(call docker_buildx,--push,debug.)
 
 .PHONY: docker-image/pyroscope/build
-docker-image/pyroscope/build: frontend/build go/bin
+docker-image/pyroscope/build: frontend/build
+	GOOS=linux $(MAKE) go/bin
 	$(call docker_buildx,--load --iidfile .docker-image-digest-pyroscope,)
 
 .PHONY: docker-image/pyroscope/push
@@ -356,7 +365,7 @@ $(BIN)/jb: Makefile go.mod
 
 $(BIN)/helm: Makefile go.mod
 	@mkdir -p $(@D)
-	GOBIN=$(abspath $(@D)) $(GO) install helm.sh/helm/v3/cmd/[email protected]
+	CGO_ENABLED=0 GOBIN=$(abspath $(@D)) $(GO) install helm.sh/helm/v3/cmd/[email protected]
 
 $(BIN)/kubeconform: Makefile go.mod
 	@mkdir -p $(@D)
@@ -379,6 +388,10 @@ $(BIN)/gotestsum: Makefile go.mod
 	@mkdir -p $(@D)
 	GOBIN=$(abspath $(@D)) $(GO) install gotest.tools/[email protected]
 
+$(BIN)/helm-docs: Makefile go.mod
+	@mkdir -p $(@D)
+	GOBIN=$(abspath $(@D)) $(GO) install github.com/norwoodj/helm-docs/cmd/[email protected]
+
 .PHONY: cve/check
 cve/check:
 	docker run -t -i --rm --volume "$(CURDIR)/:/repo" -u "$(shell id -u)" aquasec/trivy:0.45.1 filesystem --cache-dir /repo/.cache/trivy --scanners vuln --skip-dirs .tmp/ --skip-dirs node_modules/ --skip-dirs tools/monitoring/vendor/ /repo
@@ -388,8 +401,8 @@ helm/lint: $(BIN)/helm
 	$(BIN)/helm lint ./operations/pyroscope/helm/pyroscope/
 
 .PHONY: helm/docs
-helm/docs: $(BIN)/helm
-	docker run --rm --volume "$(CURDIR)/operations/pyroscope/helm:/helm-docs" -u "$(shell id -u)" jnorwood/helm-docs:v1.8.1
+helm/docs: $(BIN)/helm-docs
+	$(BIN)/helm-docs -c operations/pyroscope/helm/pyroscope
 
 .PHONY: goreleaser/lint
 goreleaser/lint: $(BIN)/goreleaser
@@ -402,14 +415,23 @@ helm/check: $(BIN)/kubeconform $(BIN)/helm
 	$(BIN)/helm dependency update ./operations/pyroscope/helm/pyroscope/
 	$(BIN)/helm dependency build ./operations/pyroscope/helm/pyroscope/
 	mkdir -p ./operations/pyroscope/helm/pyroscope/rendered/
-	$(BIN)/helm template -n default --kube-version "1.23.0" pyroscope-dev ./operations/pyroscope/helm/pyroscope/ \
+	$(BIN)/helm template -n default --kube-version "1.23.0" pyroscope-dev ./operations/pyroscope/helm/pyroscope/ $(HELM_FLAGS_V1) \
 		| tee ./operations/pyroscope/helm/pyroscope/rendered/single-binary.yaml \
 		| $(BIN)/kubeconform --summary --strict --kubernetes-version 1.23.0
-	$(BIN)/helm template -n default --kube-version "1.23.0" pyroscope-dev ./operations/pyroscope/helm/pyroscope/ --values operations/pyroscope/helm/pyroscope/values-micro-services.yaml \
+	$(BIN)/helm template -n default --kube-version "1.23.0" pyroscope-dev ./operations/pyroscope/helm/pyroscope/ $(HELM_FLAGS_V2) \
+		| tee ./operations/pyroscope/helm/pyroscope/rendered/single-binary-v2.yaml \
+		| $(BIN)/kubeconform --summary --strict --kubernetes-version 1.23.0
+	$(BIN)/helm template -n default --kube-version "1.23.0" pyroscope-dev ./operations/pyroscope/helm/pyroscope/ $(HELM_FLAGS_V1_MICROSERVICES) \
 		| tee ./operations/pyroscope/helm/pyroscope/rendered/micro-services.yaml \
 		| $(BIN)/kubeconform --summary --strict --kubernetes-version 1.23.0
+	$(BIN)/helm template -n default --kube-version "1.23.0" pyroscope-dev ./operations/pyroscope/helm/pyroscope/ $(HELM_FLAGS_V2_MICROSERVICES) \
+		| tee ./operations/pyroscope/helm/pyroscope/rendered/micro-services-v2.yaml \
+		| $(BIN)/kubeconform --summary --strict --kubernetes-version 1.23.0
+	$(BIN)/helm template -n default --kube-version "1.23.0" pyroscope-dev ./operations/pyroscope/helm/pyroscope/ --values operations/pyroscope/helm/pyroscope/values-micro-services.yaml \
+		| tee ./operations/pyroscope/helm/pyroscope/rendered/legacy-micro-services.yaml \
+		| $(BIN)/kubeconform --summary --strict --kubernetes-version 1.23.0
 	$(BIN)/helm template -n default --kube-version "1.23.0" pyroscope-dev ./operations/pyroscope/helm/pyroscope/ --values operations/pyroscope/helm/pyroscope/values-micro-services-hpa.yaml \
-		| tee ./operations/pyroscope/helm/pyroscope/rendered/micro-services-hpa.yaml \
+		| tee ./operations/pyroscope/helm/pyroscope/rendered/legacy-micro-services-hpa.yaml \
 		| $(BIN)/kubeconform --summary --strict --kubernetes-version 1.23.0
 	cat operations/pyroscope/helm/pyroscope/values-micro-services.yaml \
 		| go run ./tools/yaml-to-json \
@@ -423,15 +445,19 @@ helm/check: $(BIN)/kubeconform $(BIN)/helm
 
 .PHONY: deploy
 deploy: $(BIN)/kind $(BIN)/helm docker-image/pyroscope/build
-	$(call deploy,pyroscope-dev,)
-	# Create a service to provide the same endpoint as micro-services
-	echo '{"kind":"Service","apiVersion":"v1","metadata":{"name":"pyroscope-micro-services-query-frontend"},"spec":{"ports":[{"name":"pyroscope","port":4040,"targetPort":4040}],"selector":{"app.kubernetes.io/component":"all","app.kubernetes.io/instance":"pyroscope-dev"},"type":"ClusterIP"}}' | kubectl apply -f -
+	$(call deploy,pyroscope-dev,$(HELM_FLAGS_V2))
+
+.PHONY: deploy-v1
+deploy-v1: $(BIN)/kind $(BIN)/helm docker-image/pyroscope/build
+	$(call deploy,pyroscope-dev,$(HELM_FLAGS_V1))
 
 .PHONY: deploy-micro-services
 deploy-micro-services: $(BIN)/kind $(BIN)/helm docker-image/pyroscope/build
-	# Ensure to delete existing service, that has been created manually by the deploy target
-	kubectl delete svc --field-selector metadata.name=pyroscope-micro-services-query-frontend -l app.kubernetes.io/managed-by!=Helm || true
-	$(call deploy,pyroscope-micro-services,--values=operations/pyroscope/helm/pyroscope/values-micro-services.yaml --set pyroscope.components.querier.resources=null --set pyroscope.components.distributor.resources=null --set pyroscope.components.ingester.resources=null --set pyroscope.components.store-gateway.resources=null --set pyroscope.components.compactor.resources=null)
+	$(call deploy,pyroscope-micro-services,$(HELM_FLAGS_V2_MICROSERVICES))
+
+.PHONY: deploy-micro-services-v1
+deploy-micro-services-v1: $(BIN)/kind $(BIN)/helm docker-image/pyroscope/build
+	$(call deploy,pyroscope-micro-services,$(HELM_FLAGS_V1_MICROSERVICES))
 
 .PHONY: deploy-monitoring
 deploy-monitoring: $(BIN)/tk $(BIN)/kind tools/monitoring/environments/default/spec.json

cmd/pyroscope/Dockerfile

@@ -16,6 +16,8 @@ VOLUME /data
 COPY --chown=pyroscope:pyroscope --from=debug /data /data
 VOLUME /data-compactor
 COPY --chown=pyroscope:pyroscope --from=debug /data /data-compactor
+VOLUME /data-metastore
+COPY --chown=pyroscope:pyroscope --from=debug /data /data-metastore
 
 COPY cmd/pyroscope/pyroscope.yaml /etc/pyroscope/config.yaml
 COPY profilecli /usr/bin/profilecli

cmd/pyroscope/debug.Dockerfile

@@ -7,10 +7,9 @@ RUN addgroup -g 10001 -S pyroscope && \
 
 # This folder is created by adduser command with right owner/group
 VOLUME /data
-
-# This folder needs to be created and set to the right owner/group
 VOLUME /data-compactor
-RUN mkdir -p /data-compactor && chown pyroscope:pyroscope /data /data-compactor
+VOLUME /data-metastore
+RUN chown pyroscope:pyroscope /data /data-compactor /data-metastore
 
 COPY .tmp/bin/dlv /usr/bin/dlv
 COPY cmd/pyroscope/pyroscope.yaml /etc/pyroscope/config.yaml

docs/internal/V2-MIGRATION.md

@@ -0,0 +1,99 @@
+# Migration v1 to v2
+
+## Single binary
+
+### First deploy v1
+
+- Note: Needs persistence enabled otherwise data will be lost after restart
+
+```
+helm upgrade \
+  pyroscope \
+  ./operations/pyroscope/helm/pyroscope \
+  --install \
+  --set architecture.storage.v1=true \
+  --set architecture.storage.v2=false \
+  --set pyroscope.persistence.enabled=true
+```
+
+### Step: 2: Deploy v2 and enable dual ingest
+
+- Note: The python command will switch to the v2 read path for data ingested 10 minutes after it is run.
+
+```
+helm upgrade \
+  pyroscope \
+  ./operations/pyroscope/helm/pyroscope \
+  --set architecture.storage.v1=true \
+  --set architecture.storage.v2=true \
+  --set pyroscope.persistence.enabled=true \
+  --set architecture.storage.migration.queryBackendFrom=$(python3 -c "import datetime; print((datetime.datetime.now(datetime.UTC)+ datetime.timedelta(minutes = 10)).strftime('%Y-%m-%dT%H:%M:%SZ'))")
+```
+
+### Step 3: Remove v1 components
+
+Once data before Step 2 is no longer relevant, we can get rid of the v1 components. This will loose all data before Step 2.
+
+```
+helm upgrade \
+  pyroscope \
+  ./operations/pyroscope/helm/pyroscope \
+  --set architecture.storage.v1=false \
+  --set architecture.storage.v2=true \
+  --set pyroscope.persistence.enabled=true
+```
+
+
+## Micro-Services
+
+### First deploy v1
+
+- Note: Needs persistence enabled otherwise data will be lost after restart
+- Note:   `--set architecture.overwriteResources.requests.cpu=10m` allow this to be tested without allocation many resources, this should not be used in production
+
+
+```
+helm upgrade \
+  pyroscope \
+  ./operations/pyroscope/helm/pyroscope \
+  --install \
+  --set architecture.microservices.enabled=true \
+  --set architecture.overwriteResources.requests.cpu=10m \
+  --set minio.enabled=true \
+  --set architecture.storage.v1=true \
+  --set architecture.storage.v2=false \
+  --set pyroscope.persistence.enabled=true
+```
+
+### Step: 2: Deploy v2 and enable dual ingest
+
+- Note: The python command will switch to the v2 read path for data ingested 10 minutes after it is run.
+
+```
+helm upgrade \
+  pyroscope \
+  ./operations/pyroscope/helm/pyroscope \
+  --set architecture.microservices.enabled=true \
+  --set architecture.overwriteResources.requests.cpu=10m \
+  --set minio.enabled=true \
+  --set architecture.storage.v1=true \
+  --set architecture.storage.v2=true \
+  --set pyroscope.persistence.enabled=true \
+  --set architecture.storage.migration.queryBackendFrom=$(python3 -c "import datetime; print((datetime.datetime.now(datetime.UTC)+ datetime.timedelta(minutes = 10)).strftime('%Y-%m-%dT%H:%M:%SZ'))")
+```
+
+
+### Step 3: Remove v1 components
+
+Once data before Step 2 is no longer relevant, we can get rid of the v1 components. This will loose all data before Step 2.
+
+```
+helm upgrade \
+  pyroscope \
+  ./operations/pyroscope/helm/pyroscope \
+  --set architecture.microservices.enabled=true \
+  --set minio.enabled=true \
+  --set architecture.storage.v1=false \
+  --set architecture.storage.v2=true \
+  --set pyroscope.persistence.enabled=true
+

operations/pyroscope/helm/pyroscope/README.md

@@ -25,6 +25,16 @@
 |-----|------|---------|-------------|
 | agent | object | `{"agent":{"clustering":{"enabled":true},"configMap":{"create":false,"name":"grafana-agent-config-pyroscope"}},"controller":{"podAnnotations":{"profiles.grafana.com/cpu.port_name":"http-metrics","profiles.grafana.com/cpu.scrape":"true","profiles.grafana.com/goroutine.port_name":"http-metrics","profiles.grafana.com/goroutine.scrape":"true","profiles.grafana.com/memory.port_name":"http-metrics","profiles.grafana.com/memory.scrape":"true"},"replicas":1,"type":"statefulset"},"enabled":false}` | ----------------------------------- |
 | alloy | object | `{"alloy":{"clustering":{"enabled":true},"configMap":{"create":false,"name":"alloy-config-pyroscope"},"stabilityLevel":"public-preview"},"controller":{"podAnnotations":{"profiles.grafana.com/cpu.port_name":"http-metrics","profiles.grafana.com/cpu.scrape":"true","profiles.grafana.com/goroutine.port_name":"http-metrics","profiles.grafana.com/goroutine.scrape":"true","profiles.grafana.com/memory.port_name":"http-metrics","profiles.grafana.com/memory.scrape":"true","profiles.grafana.com/service_git_ref":"v1.8.1","profiles.grafana.com/service_repository":"https://github.com/grafana/alloy"},"replicas":1,"type":"statefulset"},"enabled":true}` | ----------------------------------- |
+| architecture.deployUnifiedServices | bool | `false` | Deploy unified write/read services. These endpoints will can be used no matter if the helm chart is configured as single-binary or microservices |
+| architecture.microservices.clusterLabelSuffix | string | `"-micro-services"` | Memberlist cluster label that will be used for all members of this cluster |
+| architecture.microservices.enabled | bool | `false` | Enable micro-services deployment mode. This is recommend for larger scale deployment and allow right size each aspect of Pyroscope. |
+| architecture.overwriteResources | object | `{}` | This flag is useful for testing, it will overwrite all pods resource statements with its contents |
+| architecture.storage.migration.ingesterWeight | float | `1` | Specifies the fraction [0:1] that should be send to the v1 write path / ingester in combined mode. 0 means no traffics is sent to ingester. 1 means 100% of requests are sent to ingester. |
+| architecture.storage.migration.queryBackend | bool | `true` | Specify a time stamp from when the v2 read path should serve traffic. |
+| architecture.storage.migration.queryBackendFrom | string | `"auto"` | Specify a time stamp from when the v2 read path should serve traffic. |
+| architecture.storage.migration.segmentWriterWeight | float | `1` | Specifies the fraction [0:1] that should be send to the v2 write path / segment-writer in combined mode. 0 means no traffics is sent to segment-writer. 1 means 100% of requests are sent to segment-writer. |
+| architecture.storage.v1 | bool | `true` | Enable v1 storage layer. |
+| architecture.storage.v2 | bool | `false` | Enable v2 storage layer. |
 | ingress.annotations | object | `{}` |  |
 | ingress.className | string | `""` |  |
 | ingress.enabled | bool | `false` |  |
@@ -45,13 +55,17 @@
 | pyroscope.extraVolumeMounts | list | `[]` |  |
 | pyroscope.extraVolumes | list | `[]` |  |
 | pyroscope.fullnameOverride | string | `""` |  |
+| pyroscope.grpc.port | int | `9095` |  |
+| pyroscope.grpc.port_name | string | `"grpc"` |  |
 | pyroscope.image.pullPolicy | string | `"IfNotPresent"` |  |
 | pyroscope.image.repository | string | `"grafana/pyroscope"` |  |
 | pyroscope.image.tag | string | `""` |  |
 | pyroscope.imagePullSecrets | list | `[]` |  |
 | pyroscope.initContainers | list | `[]` |  |
 | pyroscope.memberlist.port | int | `7946` |  |
 | pyroscope.memberlist.port_name | string | `"memberlist"` |  |
+| pyroscope.metastore.port | int | `9099` |  |
+| pyroscope.metastore.port_name | string | `"raft"` |  |
 | pyroscope.nameOverride | string | `""` |  |
 | pyroscope.nodeSelector | object | `{}` |  |
 | pyroscope.persistence.accessModes[0] | string | `"ReadWriteOnce"` |  |
@@ -97,5 +111,3 @@
 | serviceMonitor.targetLabels | list | `[]` | ServiceMonitor will add labels from the service to the Prometheus metric https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitorspec |
 | serviceMonitor.tlsConfig | string | `nil` | ServiceMonitor will use these tlsConfig settings to make the health check requests |
 
-----------------------------------------------
-Autogenerated from chart metadata using [helm-docs v1.8.1](https://github.com/norwoodj/helm-docs/releases/v1.8.1)