From ce0f8116040a068a86b35ad3347de7af7702f008 Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Tue, 28 Oct 2025 18:45:24 +0100 Subject: [PATCH 1/8] Kubernetes: add monitoring (victoria metrics) --- .../templates/globalpolicy.yaml | 2 +- .../victoria-metrics-k8s-stack/namespace.yaml | 15 ++++ .../values.yaml.gotmpl | 83 +++++++++++++++++++ .../victoria-metrics-operator/namespace.yaml | 15 ++++ .../values.yaml.gotmpl | 8 ++ charts/victoria-metrics-stack/Chart.lock | 9 ++ charts/victoria-metrics-stack/Chart.yaml | 39 +++++++++ charts/victoria-metrics-stack/namespace.yaml | 15 ++++ .../templates/networkpolicies.yaml | 40 +++++++++ .../victoria-metrics-stack/values.yaml.gotmpl | 50 +++++++++++ 10 files changed, 275 insertions(+), 1 deletion(-) create mode 100644 charts/victoria-metrics-k8s-stack/namespace.yaml create mode 100644 charts/victoria-metrics-k8s-stack/values.yaml.gotmpl create mode 100644 charts/victoria-metrics-operator/namespace.yaml create mode 100644 charts/victoria-metrics-operator/values.yaml.gotmpl create mode 100644 charts/victoria-metrics-stack/Chart.lock create mode 100644 charts/victoria-metrics-stack/Chart.yaml create mode 100644 charts/victoria-metrics-stack/namespace.yaml create mode 100644 charts/victoria-metrics-stack/templates/networkpolicies.yaml create mode 100644 charts/victoria-metrics-stack/values.yaml.gotmpl diff --git a/charts/calico-configuration/templates/globalpolicy.yaml b/charts/calico-configuration/templates/globalpolicy.yaml index 30d787f0..c1eda81f 100644 --- a/charts/calico-configuration/templates/globalpolicy.yaml +++ b/charts/calico-configuration/templates/globalpolicy.yaml @@ -8,7 +8,7 @@ spec: # "calico-system", "calico-apiserver", "tigera-operator" -- calico namespaces (when installed via scripts [local deployment]) # TODO: other namespaces are to be removed from this list (once appropriate network policies are created) namespaceSelector: - kubernetes.io/metadata.name not in {"kube-public", "kube-system", "kube-node-lease", "calico-system", "calico-apiserver", "tigera-operator", "reflector", "traefik", "victoria-logs", "csi-s3", "portainer", "topolvm", "local-path-storage", "longhorn"} + kubernetes.io/metadata.name not in {"kube-public", "kube-system", "kube-node-lease", "calico-system", "calico-apiserver", "tigera-operator", "reflector", "traefik", "victoria-logs", "csi-s3", "portainer", "topolvm", "local-path-storage", "longhorn", "victoria-metrics-operator"} types: - Ingress - Egress diff --git a/charts/victoria-metrics-k8s-stack/namespace.yaml b/charts/victoria-metrics-k8s-stack/namespace.yaml new file mode 100644 index 00000000..1534f210 --- /dev/null +++ b/charts/victoria-metrics-k8s-stack/namespace.yaml @@ -0,0 +1,15 @@ +# namespace with defined pod security standard +# inspired from https://aro-labs.com/pod-security-standards/ +# official doc: https://kubernetes.io/docs/concepts/security/pod-security-standards/ +# +# Warning: if pod / container does not meet enforced standards, it will not be deployed (silently) +# execute `kubectl -n events` to see errors (e.g.) +# Error creating: pods "xyz" is forbidden: violates PodSecurity "baseline:latest": privileged +# container "xyz" must not set securityContext.privileged to true +# +apiVersion: v1 +kind: Namespace +metadata: + name: monitoring + labels: + pod-security.kubernetes.io/enforce: restricted diff --git a/charts/victoria-metrics-k8s-stack/values.yaml.gotmpl b/charts/victoria-metrics-k8s-stack/values.yaml.gotmpl new file mode 100644 index 00000000..1bf254ca --- /dev/null +++ b/charts/victoria-metrics-k8s-stack/values.yaml.gotmpl @@ -0,0 +1,83 @@ +vmsingle: + # values documented here https://docs.victoriametrics.com/operator/api/#vmsingle + spec: + replicaCount: 2 + port: "8428" # must be string or field validation fails + useStrictSecurity: true + + # podSecurityContext: &restrictedPodSecurityContext + # enabled: true + # runAsNonRoot: true + # runAsUser: 1000 + # privileged: false + + securityContext: &restrictedSecurityContext + enabled: true + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + seccompProfile: + type: RuntimeDefault + + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: "kubernetes.io/hostname" + whenUnsatisfiable: DoNotSchedule + # hardcoded due to https://github.com/VictoriaMetrics/helm-charts/issues/2219 + labelSelector: + matchLabels: + app: server + app.kubernetes.io/instance: victoria-metrics + app.kubernetes.io/name: victoria-metrics-k8s-stack +# we manage operator and crds in separate chart +# it is easier to delete victoria metrics charts +# separately this way +victoria-metrics-operator: + enabled: false + +alertmanager: + enabled: false + +vmagent: + enabled: false + +vmalert: + enabled: false + +grafana: + enabled: false + +prometheus-node-exporter: + enabled: false + +kube-state-metrics: + enabled: false + +kubelet: + enabled: false + +kubeApiServer: + enabled: false + +kubeControllerManager: + enabled: false + +coreDns: + # -- Enabled CoreDNS metrics scraping + enabled: false + +kubeEtcd: + enabled: false + +kubeScheduler: + # -- Enable KubeScheduler metrics scraping + enabled: false + +defaultDashboards: + # -- Enable custom dashboards installation + enabled: false + +defaultRules: + # -- Enable custom alerting rules installation + create: false diff --git a/charts/victoria-metrics-operator/namespace.yaml b/charts/victoria-metrics-operator/namespace.yaml new file mode 100644 index 00000000..ec72ccbc --- /dev/null +++ b/charts/victoria-metrics-operator/namespace.yaml @@ -0,0 +1,15 @@ +# namespace with defined pod security standard +# inspired from https://aro-labs.com/pod-security-standards/ +# official doc: https://kubernetes.io/docs/concepts/security/pod-security-standards/ +# +# Warning: if pod / container does not meet enforced standards, it will not be deployed (silently) +# execute `kubectl -n events` to see errors (e.g.) +# Error creating: pods "xyz" is forbidden: violates PodSecurity "baseline:latest": privileged +# container "xyz" must not set securityContext.privileged to true +# +apiVersion: v1 +kind: Namespace +metadata: + name: victoria-metrics-operator + labels: + pod-security.kubernetes.io/enforce: restricted diff --git a/charts/victoria-metrics-operator/values.yaml.gotmpl b/charts/victoria-metrics-operator/values.yaml.gotmpl new file mode 100644 index 00000000..86e63505 --- /dev/null +++ b/charts/victoria-metrics-operator/values.yaml.gotmpl @@ -0,0 +1,8 @@ +securityContext: + seccompProfile: + type: RuntimeDefault + +admissionWebhooks: + certManager: + # avoid new cert generation on every helm run + enabled: true diff --git a/charts/victoria-metrics-stack/Chart.lock b/charts/victoria-metrics-stack/Chart.lock new file mode 100644 index 00000000..29253b1e --- /dev/null +++ b/charts/victoria-metrics-stack/Chart.lock @@ -0,0 +1,9 @@ +dependencies: +- name: victoria-metrics-single + repository: https://victoriametrics.github.io/helm-charts/ + version: 0.25.2 +- name: victoria-metrics-agent + repository: https://victoriametrics.github.io/helm-charts/ + version: 0.26.2 +digest: sha256:e9a8c4ed4495ecfcf9962a6aa7fc9f6a6e8813e69a20daa9bb38b2d9a018c50e +generated: "2025-10-26T12:42:21.158234622+01:00" diff --git a/charts/victoria-metrics-stack/Chart.yaml b/charts/victoria-metrics-stack/Chart.yaml new file mode 100644 index 00000000..1b7f87f1 --- /dev/null +++ b/charts/victoria-metrics-stack/Chart.yaml @@ -0,0 +1,39 @@ +apiVersion: v2 +name: victoria-metrics-stack +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.128.0" + +dependencies: + - name: victoria-metrics-single + version: 0.25.2 + repository: &victoria-metrics-repo "https://victoriametrics.github.io/helm-charts/" + condition: victoria-metrics-single.enabled + + # - name: victoria-metrics-auth + # version: 0.19.7 + # repository: *victoria-metrics-repo + + - name: victoria-metrics-agent + version: 0.26.2 + repository: *victoria-metrics-repo + condition: victoria-metrics-agent.enabled diff --git a/charts/victoria-metrics-stack/namespace.yaml b/charts/victoria-metrics-stack/namespace.yaml new file mode 100644 index 00000000..1534f210 --- /dev/null +++ b/charts/victoria-metrics-stack/namespace.yaml @@ -0,0 +1,15 @@ +# namespace with defined pod security standard +# inspired from https://aro-labs.com/pod-security-standards/ +# official doc: https://kubernetes.io/docs/concepts/security/pod-security-standards/ +# +# Warning: if pod / container does not meet enforced standards, it will not be deployed (silently) +# execute `kubectl -n events` to see errors (e.g.) +# Error creating: pods "xyz" is forbidden: violates PodSecurity "baseline:latest": privileged +# container "xyz" must not set securityContext.privileged to true +# +apiVersion: v1 +kind: Namespace +metadata: + name: monitoring + labels: + pod-security.kubernetes.io/enforce: restricted diff --git a/charts/victoria-metrics-stack/templates/networkpolicies.yaml b/charts/victoria-metrics-stack/templates/networkpolicies.yaml new file mode 100644 index 00000000..56d8ab81 --- /dev/null +++ b/charts/victoria-metrics-stack/templates/networkpolicies.yaml @@ -0,0 +1,40 @@ +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: vm-server-network-policy +spec: + selector: >- + app.kubernetes.io/name == "victoria-metrics-single" + && app.kubernetes.io/instance == "{{ .Release.Name }}" + ingress: + - action: Allow + protocol: TCP + destination: + ports: + - {{ index .Values "victoria-metrics-single" "server" "service" "servicePort" }} + +--- + +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: vm-agent-network-policy +spec: + selector: >- + app.kubernetes.io/name == "victoria-metrics-agent" + && app.kubernetes.io/instance == "{{ .Release.Name }}" + egress: + - action: Allow + protocol: TCP + destination: + nets: + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16 + ports: + - 6443 + - action: Allow + protocol: TCP + destination: + ports: + - {{ index .Values "victoria-metrics-single" "server" "service" "servicePort" }} diff --git a/charts/victoria-metrics-stack/values.yaml.gotmpl b/charts/victoria-metrics-stack/values.yaml.gotmpl new file mode 100644 index 00000000..f7c385c0 --- /dev/null +++ b/charts/victoria-metrics-stack/values.yaml.gotmpl @@ -0,0 +1,50 @@ +victoria-metrics-single: + enabled: true + + server: + replicaCount: 2 + + service: + servicePort: 8428 + + mode: statefulSet + + # avoid name to long (>63 char) error + fullnameOverride: vm-server + + podSecurityContext: &restrictedPodSecurityContext + enabled: true + runAsNonRoot: true + runAsUser: 1000 + privileged: false + + securityContext: &restrictedSecurityContext + enabled: true + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + seccompProfile: + type: RuntimeDefault + +victoria-metrics-agent: + enabled: true + fullnameOverride: vm-agent + + config: + global: + scrape_interval: 20s + + service: + enabled: true + servicePort: 8429 + + remoteWrite: + - url: "http://vm-server-0.vm-server.{{ .Release.Namespace }}.svc.cluster.local:8428/api/v1/write" + - url: "http://vm-server-1.vm-server.{{ .Release.Namespace }}.svc.cluster.local:8428/api/v1/write" + + podSecurityContext: *restrictedPodSecurityContext + securityContext: *restrictedSecurityContext + + +victoria-metrics-auth: From 6d20c40047a16ee4b07b2c2be47c3e6595f0dd87 Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Wed, 29 Oct 2025 16:04:46 +0100 Subject: [PATCH 2/8] Add network policy for vm operator --- .../templates/globalpolicy.yaml | 2 +- .../values.yaml.gotmpl | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/charts/calico-configuration/templates/globalpolicy.yaml b/charts/calico-configuration/templates/globalpolicy.yaml index c1eda81f..30d787f0 100644 --- a/charts/calico-configuration/templates/globalpolicy.yaml +++ b/charts/calico-configuration/templates/globalpolicy.yaml @@ -8,7 +8,7 @@ spec: # "calico-system", "calico-apiserver", "tigera-operator" -- calico namespaces (when installed via scripts [local deployment]) # TODO: other namespaces are to be removed from this list (once appropriate network policies are created) namespaceSelector: - kubernetes.io/metadata.name not in {"kube-public", "kube-system", "kube-node-lease", "calico-system", "calico-apiserver", "tigera-operator", "reflector", "traefik", "victoria-logs", "csi-s3", "portainer", "topolvm", "local-path-storage", "longhorn", "victoria-metrics-operator"} + kubernetes.io/metadata.name not in {"kube-public", "kube-system", "kube-node-lease", "calico-system", "calico-apiserver", "tigera-operator", "reflector", "traefik", "victoria-logs", "csi-s3", "portainer", "topolvm", "local-path-storage", "longhorn"} types: - Ingress - Egress diff --git a/charts/victoria-metrics-operator/values.yaml.gotmpl b/charts/victoria-metrics-operator/values.yaml.gotmpl index 86e63505..988366d5 100644 --- a/charts/victoria-metrics-operator/values.yaml.gotmpl +++ b/charts/victoria-metrics-operator/values.yaml.gotmpl @@ -6,3 +6,20 @@ admissionWebhooks: certManager: # avoid new cert generation on every helm run enabled: true + +extraObjects: +- apiVersion: projectcalico.org/v3 + kind: NetworkPolicy + metadata: + name: victoria-metrics-operator-network-policy + spec: + egress: + - action: Allow + protocol: TCP + destination: + nets: + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16 + ports: + - 6443 From 3e83321884a4667c77b564ffe09f2d15723a2ac8 Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Wed, 29 Oct 2025 16:51:19 +0100 Subject: [PATCH 3/8] Add vm auth (read loadbalancer) --- charts/traefik/values.common.yaml.gotmpl | 10 ++++++ charts/victoria-metrics-stack/Chart.lock | 7 ++-- charts/victoria-metrics-stack/Chart.yaml | 9 ++--- charts/victoria-metrics-stack/namespace.yaml | 2 +- .../vm-agent.yaml} | 17 --------- .../templates/networkpolicies/vm-auth.yaml | 20 +++++++++++ .../templates/networkpolicies/vm-server.yaml | 14 ++++++++ .../victoria-metrics-stack/values.yaml.gotmpl | 36 +++++++++++++++++++ 8 files changed, 91 insertions(+), 24 deletions(-) rename charts/victoria-metrics-stack/templates/{networkpolicies.yaml => networkpolicies/vm-agent.yaml} (58%) create mode 100644 charts/victoria-metrics-stack/templates/networkpolicies/vm-auth.yaml create mode 100644 charts/victoria-metrics-stack/templates/networkpolicies/vm-server.yaml diff --git a/charts/traefik/values.common.yaml.gotmpl b/charts/traefik/values.common.yaml.gotmpl index 330d49bf..a8b0c9b5 100644 --- a/charts/traefik/values.common.yaml.gotmpl +++ b/charts/traefik/values.common.yaml.gotmpl @@ -133,6 +133,16 @@ extraObjects: - 172.16.0.0/12 - 192.168.0.0/16 +- apiVersion: traefik.io/v1alpha1 + kind: Middleware + metadata: + name: metrics-strip-prefix + namespace: {{.Release.Namespace}} + spec: + stripPrefix: + prefixes: + - /metrics + - apiVersion: networking.k8s.io/v1 kind: Ingress metadata: diff --git a/charts/victoria-metrics-stack/Chart.lock b/charts/victoria-metrics-stack/Chart.lock index 29253b1e..ae71d5a3 100644 --- a/charts/victoria-metrics-stack/Chart.lock +++ b/charts/victoria-metrics-stack/Chart.lock @@ -2,8 +2,11 @@ dependencies: - name: victoria-metrics-single repository: https://victoriametrics.github.io/helm-charts/ version: 0.25.2 +- name: victoria-metrics-auth + repository: https://victoriametrics.github.io/helm-charts/ + version: 0.19.7 - name: victoria-metrics-agent repository: https://victoriametrics.github.io/helm-charts/ version: 0.26.2 -digest: sha256:e9a8c4ed4495ecfcf9962a6aa7fc9f6a6e8813e69a20daa9bb38b2d9a018c50e -generated: "2025-10-26T12:42:21.158234622+01:00" +digest: sha256:1b9f1ec96dee105d9ac83f78883e6ee5b8558fad9bac4e41b71d37a69dd5c745 +generated: "2025-10-29T15:55:10.919914456+01:00" diff --git a/charts/victoria-metrics-stack/Chart.yaml b/charts/victoria-metrics-stack/Chart.yaml index 1b7f87f1..faff89cb 100644 --- a/charts/victoria-metrics-stack/Chart.yaml +++ b/charts/victoria-metrics-stack/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 +version: 0.0.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -29,9 +29,10 @@ dependencies: repository: &victoria-metrics-repo "https://victoriametrics.github.io/helm-charts/" condition: victoria-metrics-single.enabled - # - name: victoria-metrics-auth - # version: 0.19.7 - # repository: *victoria-metrics-repo + - name: victoria-metrics-auth + version: 0.19.7 + repository: *victoria-metrics-repo + condition: victoria-metrics-auth.enabled - name: victoria-metrics-agent version: 0.26.2 diff --git a/charts/victoria-metrics-stack/namespace.yaml b/charts/victoria-metrics-stack/namespace.yaml index 1534f210..86c84def 100644 --- a/charts/victoria-metrics-stack/namespace.yaml +++ b/charts/victoria-metrics-stack/namespace.yaml @@ -10,6 +10,6 @@ apiVersion: v1 kind: Namespace metadata: - name: monitoring + name: victoria-metrics-stack labels: pod-security.kubernetes.io/enforce: restricted diff --git a/charts/victoria-metrics-stack/templates/networkpolicies.yaml b/charts/victoria-metrics-stack/templates/networkpolicies/vm-agent.yaml similarity index 58% rename from charts/victoria-metrics-stack/templates/networkpolicies.yaml rename to charts/victoria-metrics-stack/templates/networkpolicies/vm-agent.yaml index 56d8ab81..a27788af 100644 --- a/charts/victoria-metrics-stack/templates/networkpolicies.yaml +++ b/charts/victoria-metrics-stack/templates/networkpolicies/vm-agent.yaml @@ -1,20 +1,3 @@ -apiVersion: projectcalico.org/v3 -kind: NetworkPolicy -metadata: - name: vm-server-network-policy -spec: - selector: >- - app.kubernetes.io/name == "victoria-metrics-single" - && app.kubernetes.io/instance == "{{ .Release.Name }}" - ingress: - - action: Allow - protocol: TCP - destination: - ports: - - {{ index .Values "victoria-metrics-single" "server" "service" "servicePort" }} - ---- - apiVersion: projectcalico.org/v3 kind: NetworkPolicy metadata: diff --git a/charts/victoria-metrics-stack/templates/networkpolicies/vm-auth.yaml b/charts/victoria-metrics-stack/templates/networkpolicies/vm-auth.yaml new file mode 100644 index 00000000..efee32cf --- /dev/null +++ b/charts/victoria-metrics-stack/templates/networkpolicies/vm-auth.yaml @@ -0,0 +1,20 @@ +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: vm-auth-network-policy +spec: + selector: >- + app.kubernetes.io/name == "victoria-metrics-auth" + && app.kubernetes.io/instance == "{{ .Release.Name }}" + ingress: + - action: Allow + protocol: TCP + destination: + ports: + - {{ index .Values "victoria-metrics-auth" "service" "servicePort" }} + egress: + - action: Allow + protocol: TCP + destination: + ports: + - {{ index .Values "victoria-metrics-single" "server" "service" "servicePort" }} diff --git a/charts/victoria-metrics-stack/templates/networkpolicies/vm-server.yaml b/charts/victoria-metrics-stack/templates/networkpolicies/vm-server.yaml new file mode 100644 index 00000000..3a2758d6 --- /dev/null +++ b/charts/victoria-metrics-stack/templates/networkpolicies/vm-server.yaml @@ -0,0 +1,14 @@ +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: vm-server-network-policy +spec: + selector: >- + app.kubernetes.io/name == "victoria-metrics-single" + && app.kubernetes.io/instance == "{{ .Release.Name }}" + ingress: + - action: Allow + protocol: TCP + destination: + ports: + - {{ index .Values "victoria-metrics-single" "server" "service" "servicePort" }} diff --git a/charts/victoria-metrics-stack/values.yaml.gotmpl b/charts/victoria-metrics-stack/values.yaml.gotmpl index f7c385c0..72d60b9f 100644 --- a/charts/victoria-metrics-stack/values.yaml.gotmpl +++ b/charts/victoria-metrics-stack/values.yaml.gotmpl @@ -48,3 +48,39 @@ victoria-metrics-agent: victoria-metrics-auth: + enabled: true + fullnameOverride: vm-auth + + service: + servicePort: 8427 + + ingress: + enabled: true + annotations: + namespace: {{ .Release.Namespace }} + traefik.ingress.kubernetes.io/router.tls: "true" + traefik.ingress.kubernetes.io/router.middlewares: traefik-metrics-strip-prefix@kubernetescrd,traefik-traefik-basic-auth@kubernetescrd + traefik.ingress.kubernetes.io/router.entrypoints: websecure + hosts: + - name: {{ requiredEnv "K8S_MONITORING_FQDN" }} + path: + - /metrics + port: http + + podSecurityContext: *restrictedPodSecurityContext + securityContext: *restrictedSecurityContext + + resources: + limits: + cpu: 0.5 + memory: 256Mi + requests: + cpu: 100m + memory: 128Mi + + config: + unauthorized_user: + url_prefix: + - "http://vm-server-0.vm-server.{{ .Release.Namespace }}.svc.cluster.local:8428/" + - "http://vm-server-1.vm-server.{{ .Release.Namespace }}.svc.cluster.local:8428/" + load_balancing_policy: first_available From d84f2d4404b2112d92ed026b19cf3259d216007f Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Wed, 29 Oct 2025 20:12:53 +0100 Subject: [PATCH 4/8] VM Operator: disable prometheus converter --- charts/victoria-metrics-operator/values.yaml.gotmpl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/charts/victoria-metrics-operator/values.yaml.gotmpl b/charts/victoria-metrics-operator/values.yaml.gotmpl index 988366d5..46900c4c 100644 --- a/charts/victoria-metrics-operator/values.yaml.gotmpl +++ b/charts/victoria-metrics-operator/values.yaml.gotmpl @@ -1,3 +1,6 @@ +operator: + disable_prometheus_converter: true + securityContext: seccompProfile: type: RuntimeDefault From f312e65eff0e2a83d792dac304d2183d449474e2 Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Wed, 29 Oct 2025 20:13:24 +0100 Subject: [PATCH 5/8] VM Agent: add resource constraints --- charts/victoria-metrics-stack/values.yaml.gotmpl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/charts/victoria-metrics-stack/values.yaml.gotmpl b/charts/victoria-metrics-stack/values.yaml.gotmpl index 72d60b9f..3b183e13 100644 --- a/charts/victoria-metrics-stack/values.yaml.gotmpl +++ b/charts/victoria-metrics-stack/values.yaml.gotmpl @@ -46,6 +46,13 @@ victoria-metrics-agent: podSecurityContext: *restrictedPodSecurityContext securityContext: *restrictedSecurityContext + resources: + limits: + cpu: 2 + memory: 1Gi + requests: + cpu: 0.5 + memory: 256Mi victoria-metrics-auth: enabled: true From b7f9b597304ef626248c628efe6e8fbb0af5cfc8 Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Thu, 30 Oct 2025 08:39:51 +0100 Subject: [PATCH 6/8] Automatically add `/` to /metrics path --- charts/traefik/values.common.yaml.gotmpl | 11 +++++++++++ charts/victoria-metrics-stack/values.yaml.gotmpl | 5 ++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/charts/traefik/values.common.yaml.gotmpl b/charts/traefik/values.common.yaml.gotmpl index a8b0c9b5..936ffb86 100644 --- a/charts/traefik/values.common.yaml.gotmpl +++ b/charts/traefik/values.common.yaml.gotmpl @@ -143,6 +143,17 @@ extraObjects: prefixes: - /metrics + # a (href) links do not work properly without trailing slash +- apiVersion: traefik.io/v1alpha1 + kind: Middleware + metadata: + name: metrics-path-append-slash + namespace: {{ .Release.Namespace }} + spec: + redirectRegex: + regex: "^(https?://[^/]+/metrics)$" + replacement: "${1}/" + - apiVersion: networking.k8s.io/v1 kind: Ingress metadata: diff --git a/charts/victoria-metrics-stack/values.yaml.gotmpl b/charts/victoria-metrics-stack/values.yaml.gotmpl index 3b183e13..dfeba64a 100644 --- a/charts/victoria-metrics-stack/values.yaml.gotmpl +++ b/charts/victoria-metrics-stack/values.yaml.gotmpl @@ -66,7 +66,10 @@ victoria-metrics-auth: annotations: namespace: {{ .Release.Namespace }} traefik.ingress.kubernetes.io/router.tls: "true" - traefik.ingress.kubernetes.io/router.middlewares: traefik-metrics-strip-prefix@kubernetescrd,traefik-traefik-basic-auth@kubernetescrd + traefik.ingress.kubernetes.io/router.middlewares: >- + traefik-metrics-path-append-slash@kubernetescrd, + traefik-metrics-strip-prefix@kubernetescrd, + traefik-traefik-basic-auth@kubernetescrd traefik.ingress.kubernetes.io/router.entrypoints: websecure hosts: - name: {{ requiredEnv "K8S_MONITORING_FQDN" }} From 4433781368864ae7fe3ea0d4569f09f608762699 Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Sun, 2 Nov 2025 11:51:52 +0100 Subject: [PATCH 7/8] update --- charts/Makefile | 5 - .../values.yaml.gotmpl | 19 +- charts/victoria-metrics-stack/Chart.lock | 14 +- charts/victoria-metrics-stack/Chart.yaml | 22 +-- .../networkpolicies/vm-agent.yaml | 0 .../networkpolicies/vm-auth.yaml | 0 .../networkpolicies/vm-server.yaml | 0 .../templates/vmsingle.yaml | 17 ++ .../values.old.yaml.gotmpl | 96 ++++++++++ .../victoria-metrics-stack/values.yaml.gotmpl | 173 ++++++++---------- 10 files changed, 215 insertions(+), 131 deletions(-) rename charts/victoria-metrics-stack/{templates => }/networkpolicies/vm-agent.yaml (100%) rename charts/victoria-metrics-stack/{templates => }/networkpolicies/vm-auth.yaml (100%) rename charts/victoria-metrics-stack/{templates => }/networkpolicies/vm-server.yaml (100%) create mode 100644 charts/victoria-metrics-stack/templates/vmsingle.yaml create mode 100644 charts/victoria-metrics-stack/values.old.yaml.gotmpl diff --git a/charts/Makefile b/charts/Makefile index 391e44f3..6b63312e 100644 --- a/charts/Makefile +++ b/charts/Makefile @@ -52,11 +52,6 @@ helmfile-diff: .check-helmfile-installed helmfile.yaml ## Shows the differences @set -a; source $(REPO_CONFIG_LOCATION); set +a; \ $(HELMFILE) -f $(REPO_BASE_DIR)/charts/helmfile.yaml diff -.PHONY: helmfile-delete -helmfile-delete: .check-helmfile-installed helmfile.yaml ## Deletes the helmfile configuration - @set -a; source $(REPO_CONFIG_LOCATION); set +a; \ - $(HELMFILE) -f $(REPO_BASE_DIR)/charts/helmfile.yaml delete - .PHONY: up up: helmfile-apply ## Start the stack diff --git a/charts/victoria-metrics-k8s-stack/values.yaml.gotmpl b/charts/victoria-metrics-k8s-stack/values.yaml.gotmpl index 1bf254ca..2b63245a 100644 --- a/charts/victoria-metrics-k8s-stack/values.yaml.gotmpl +++ b/charts/victoria-metrics-k8s-stack/values.yaml.gotmpl @@ -1,7 +1,7 @@ vmsingle: # values documented here https://docs.victoriametrics.com/operator/api/#vmsingle spec: - replicaCount: 2 + replicaCount: 1 port: "8428" # must be string or field validation fails useStrictSecurity: true @@ -11,14 +11,14 @@ vmsingle: # runAsUser: 1000 # privileged: false - securityContext: &restrictedSecurityContext - enabled: true - capabilities: - drop: ["ALL"] - readOnlyRootFilesystem: true - allowPrivilegeEscalation: false - seccompProfile: - type: RuntimeDefault + # securityContext: &restrictedSecurityContext + # enabled: true + # capabilities: + # drop: ["ALL"] + # readOnlyRootFilesystem: true + # allowPrivilegeEscalation: false + # seccompProfile: + # type: RuntimeDefault topologySpreadConstraints: - maxSkew: 1 @@ -30,6 +30,7 @@ vmsingle: app: server app.kubernetes.io/instance: victoria-metrics app.kubernetes.io/name: victoria-metrics-k8s-stack + # we manage operator and crds in separate chart # it is easier to delete victoria metrics charts # separately this way diff --git a/charts/victoria-metrics-stack/Chart.lock b/charts/victoria-metrics-stack/Chart.lock index ae71d5a3..a1f2915e 100644 --- a/charts/victoria-metrics-stack/Chart.lock +++ b/charts/victoria-metrics-stack/Chart.lock @@ -1,12 +1,6 @@ dependencies: -- name: victoria-metrics-single +- name: victoria-metrics-k8s-stack repository: https://victoriametrics.github.io/helm-charts/ - version: 0.25.2 -- name: victoria-metrics-auth - repository: https://victoriametrics.github.io/helm-charts/ - version: 0.19.7 -- name: victoria-metrics-agent - repository: https://victoriametrics.github.io/helm-charts/ - version: 0.26.2 -digest: sha256:1b9f1ec96dee105d9ac83f78883e6ee5b8558fad9bac4e41b71d37a69dd5c745 -generated: "2025-10-29T15:55:10.919914456+01:00" + version: 0.62.0 +digest: sha256:1bf510e968425917526abc435f4f69961cd0a940cc0f59af3a2a4f0107e9e549 +generated: "2025-11-01T10:25:28.465780639+01:00" diff --git a/charts/victoria-metrics-stack/Chart.yaml b/charts/victoria-metrics-stack/Chart.yaml index faff89cb..be3af0d5 100644 --- a/charts/victoria-metrics-stack/Chart.yaml +++ b/charts/victoria-metrics-stack/Chart.yaml @@ -24,17 +24,17 @@ version: 0.0.1 appVersion: "1.128.0" dependencies: - - name: victoria-metrics-single - version: 0.25.2 - repository: &victoria-metrics-repo "https://victoriametrics.github.io/helm-charts/" + - name: victoria-metrics-k8s-stack + version: 0.62.0 + repository: "https://victoriametrics.github.io/helm-charts/" condition: victoria-metrics-single.enabled - - name: victoria-metrics-auth - version: 0.19.7 - repository: *victoria-metrics-repo - condition: victoria-metrics-auth.enabled +# - name: victoria-metrics-auth +# version: 0.19.7 +# repository: *victoria-metrics-repo +# condition: victoria-metrics-auth.enabled - - name: victoria-metrics-agent - version: 0.26.2 - repository: *victoria-metrics-repo - condition: victoria-metrics-agent.enabled +# - name: victoria-metrics-agent +# version: 0.26.2 +# repository: *victoria-metrics-repo +# condition: victoria-metrics-agent.enabled diff --git a/charts/victoria-metrics-stack/templates/networkpolicies/vm-agent.yaml b/charts/victoria-metrics-stack/networkpolicies/vm-agent.yaml similarity index 100% rename from charts/victoria-metrics-stack/templates/networkpolicies/vm-agent.yaml rename to charts/victoria-metrics-stack/networkpolicies/vm-agent.yaml diff --git a/charts/victoria-metrics-stack/templates/networkpolicies/vm-auth.yaml b/charts/victoria-metrics-stack/networkpolicies/vm-auth.yaml similarity index 100% rename from charts/victoria-metrics-stack/templates/networkpolicies/vm-auth.yaml rename to charts/victoria-metrics-stack/networkpolicies/vm-auth.yaml diff --git a/charts/victoria-metrics-stack/templates/networkpolicies/vm-server.yaml b/charts/victoria-metrics-stack/networkpolicies/vm-server.yaml similarity index 100% rename from charts/victoria-metrics-stack/templates/networkpolicies/vm-server.yaml rename to charts/victoria-metrics-stack/networkpolicies/vm-server.yaml diff --git a/charts/victoria-metrics-stack/templates/vmsingle.yaml b/charts/victoria-metrics-stack/templates/vmsingle.yaml new file mode 100644 index 00000000..de69f81a --- /dev/null +++ b/charts/victoria-metrics-stack/templates/vmsingle.yaml @@ -0,0 +1,17 @@ +{{ range $i, $e := until ( .Values.vmsingle.instanceCount | int ) }} +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMSingle +metadata: + name: vmsingle-{{ $i }} + namespace: {{ $.Release.Namespace }} + labels: + app.kubernetes.io/name: vmsingle + app.kubernetes.io/instance: vmsingle-{{ $i }} + app.kubernetes.io/component: monitoring + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: {{ $.Release.Name }} + app.kubernetes.io/version: {{ $.Values.vmsingle.spec.image.tag }} + helm.sh/chart: {{ $.Chart.Name }}-{{ $.Chart.Version }} +spec: {{ toYaml $.Values.vmsingle.spec | nindent 2 }} +--- +{{- end }} diff --git a/charts/victoria-metrics-stack/values.old.yaml.gotmpl b/charts/victoria-metrics-stack/values.old.yaml.gotmpl new file mode 100644 index 00000000..fcf4b955 --- /dev/null +++ b/charts/victoria-metrics-stack/values.old.yaml.gotmpl @@ -0,0 +1,96 @@ +victoria-metrics-single: + enabled: false + + server: + replicaCount: 2 + + service: + servicePort: 8428 + + mode: statefulSet + + # avoid name to long (>63 char) error + fullnameOverride: vm-server + + podSecurityContext: &restrictedPodSecurityContext + enabled: true + runAsNonRoot: true + runAsUser: 1000 + privileged: false + + securityContext: &restrictedSecurityContext + enabled: true + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + seccompProfile: + type: RuntimeDefault + +victoria-metrics-agent: + enabled: false + fullnameOverride: vm-agent + + config: + global: + scrape_interval: 20s + + service: + enabled: true + servicePort: 8429 + + remoteWrite: + - url: "http://vm-server-0.vm-server.{{ .Release.Namespace }}.svc.cluster.local:8428/api/v1/write" + - url: "http://vm-server-1.vm-server.{{ .Release.Namespace }}.svc.cluster.local:8428/api/v1/write" + + podSecurityContext: *restrictedPodSecurityContext + securityContext: *restrictedSecurityContext + + resources: + limits: + cpu: 2 + memory: 1Gi + requests: + cpu: 0.5 + memory: 256Mi + +victoria-metrics-auth: + enabled: false + fullnameOverride: vm-auth + + service: + servicePort: 8427 + + ingress: + enabled: true + annotations: + namespace: {{ .Release.Namespace }} + traefik.ingress.kubernetes.io/router.tls: "true" + traefik.ingress.kubernetes.io/router.middlewares: >- + traefik-metrics-path-append-slash@kubernetescrd, + traefik-metrics-strip-prefix@kubernetescrd, + traefik-traefik-basic-auth@kubernetescrd + traefik.ingress.kubernetes.io/router.entrypoints: websecure + hosts: + - name: {{ requiredEnv "K8S_MONITORING_FQDN" }} + path: + - /metrics + port: http + + podSecurityContext: *restrictedPodSecurityContext + securityContext: *restrictedSecurityContext + + resources: + limits: + cpu: 0.5 + memory: 256Mi + requests: + cpu: 100m + memory: 128Mi + + config: + unauthorized_user: + url_prefix: + - "http://vm-server-0.vm-server.{{ .Release.Namespace }}.svc.cluster.local:8428/" + - "http://vm-server-1.vm-server.{{ .Release.Namespace }}.svc.cluster.local:8428/" + load_balancing_policy: first_available diff --git a/charts/victoria-metrics-stack/values.yaml.gotmpl b/charts/victoria-metrics-stack/values.yaml.gotmpl index dfeba64a..79e23292 100644 --- a/charts/victoria-metrics-stack/values.yaml.gotmpl +++ b/charts/victoria-metrics-stack/values.yaml.gotmpl @@ -1,96 +1,77 @@ -victoria-metrics-single: - enabled: true - - server: - replicaCount: 2 - - service: - servicePort: 8428 - - mode: statefulSet - - # avoid name to long (>63 char) error - fullnameOverride: vm-server - - podSecurityContext: &restrictedPodSecurityContext - enabled: true - runAsNonRoot: true - runAsUser: 1000 - privileged: false - - securityContext: &restrictedSecurityContext - enabled: true - capabilities: - drop: ["ALL"] - readOnlyRootFilesystem: true - allowPrivilegeEscalation: false - seccompProfile: - type: RuntimeDefault - -victoria-metrics-agent: - enabled: true - fullnameOverride: vm-agent - - config: - global: - scrape_interval: 20s - - service: - enabled: true - servicePort: 8429 - - remoteWrite: - - url: "http://vm-server-0.vm-server.{{ .Release.Namespace }}.svc.cluster.local:8428/api/v1/write" - - url: "http://vm-server-1.vm-server.{{ .Release.Namespace }}.svc.cluster.local:8428/api/v1/write" - - podSecurityContext: *restrictedPodSecurityContext - securityContext: *restrictedSecurityContext - - resources: - limits: - cpu: 2 - memory: 1Gi - requests: - cpu: 0.5 - memory: 256Mi - -victoria-metrics-auth: - enabled: true - fullnameOverride: vm-auth - - service: - servicePort: 8427 - - ingress: - enabled: true - annotations: - namespace: {{ .Release.Namespace }} - traefik.ingress.kubernetes.io/router.tls: "true" - traefik.ingress.kubernetes.io/router.middlewares: >- - traefik-metrics-path-append-slash@kubernetescrd, - traefik-metrics-strip-prefix@kubernetescrd, - traefik-traefik-basic-auth@kubernetescrd - traefik.ingress.kubernetes.io/router.entrypoints: websecure - hosts: - - name: {{ requiredEnv "K8S_MONITORING_FQDN" }} - path: - - /metrics - port: http - - podSecurityContext: *restrictedPodSecurityContext - securityContext: *restrictedSecurityContext - - resources: - limits: - cpu: 0.5 - memory: 256Mi - requests: - cpu: 100m - memory: 128Mi - - config: - unauthorized_user: - url_prefix: - - "http://vm-server-0.vm-server.{{ .Release.Namespace }}.svc.cluster.local:8428/" - - "http://vm-server-1.vm-server.{{ .Release.Namespace }}.svc.cluster.local:8428/" - load_balancing_policy: first_available +vmsingle: + instanceCount: 2 # number of separate / independent vmsingle servers` + spec: # values documented here https://docs.victoriametrics.com/operator/api/#vmsingle + image: + tag: v1.128.0 + port: "8428" # must be string or field validation fails + useStrictSecurity: true + podMetadata: + labels: + app.kubernetes.io/name: vmsingle + app.kubernetes.io/part-of: {{ .Release.Name }} + + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: "kubernetes.io/hostname" + whenUnsatisfiable: DoNotSchedule + # hardcoded due to https://github.com/VictoriaMetrics/helm-charts/issues/2219 + labelSelector: + matchLabels: + app.kubernetes.io/name: vmsingle + app.kubernetes.io/part-of: {{ .Release.Name }} + +victoria-metrics-k8s-stack: + vmsingle: + enabled: false + + # we manage operator and crds in separate chart + # it is easier to delete victoria metrics charts + # separately this way + victoria-metrics-operator: + enabled: false + + alertmanager: + enabled: false + + vmagent: + enabled: false + + vmalert: + enabled: false + + grafana: + enabled: false + + prometheus-node-exporter: + enabled: false + + kube-state-metrics: + enabled: false + + kubelet: + enabled: false + + kubeApiServer: + enabled: false + + kubeControllerManager: + enabled: false + + coreDns: + # -- Enabled CoreDNS metrics scraping + enabled: false + + kubeEtcd: + enabled: false + + kubeScheduler: + # -- Enable KubeScheduler metrics scraping + enabled: false + + defaultDashboards: + # -- Enable custom dashboards installation + enabled: false + + defaultRules: + # -- Enable custom alerting rules installation + create: false From f2101cfe19410cb9832f34150518956ee393b23a Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Sun, 2 Nov 2025 12:24:40 +0100 Subject: [PATCH 8/8] Document limitation with 1 node cluster --- charts/victoria-metrics-stack/values.yaml.gotmpl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/charts/victoria-metrics-stack/values.yaml.gotmpl b/charts/victoria-metrics-stack/values.yaml.gotmpl index 79e23292..528da2e1 100644 --- a/charts/victoria-metrics-stack/values.yaml.gotmpl +++ b/charts/victoria-metrics-stack/values.yaml.gotmpl @@ -10,11 +10,12 @@ vmsingle: app.kubernetes.io/name: vmsingle app.kubernetes.io/part-of: {{ .Release.Name }} + # does not work in 1-node clusters + # https://github.com/kubernetes/kubernetes/issues/105072 topologySpreadConstraints: - maxSkew: 1 topologyKey: "kubernetes.io/hostname" whenUnsatisfiable: DoNotSchedule - # hardcoded due to https://github.com/VictoriaMetrics/helm-charts/issues/2219 labelSelector: matchLabels: app.kubernetes.io/name: vmsingle