Handling the uninstallation properly + target allocator fixes

Signed-off-by: Jirka Kremser <jiri.kremser@gmail.com>

Author: jkremser

.gitignore

@@ -8,6 +8,7 @@ dist/
 
 # Helm charts
 helmchart/otel-add-on/charts/*.tgz
+helmchart/otel-add-on/*.tgz
 
 # generated
 /generated/*

helmchart/otel-add-on/.helmignore

@@ -21,3 +21,7 @@
 .idea/
 *.tmproj
 .vscode/
+
+# Custom
+otel-add-on*.tgz
+_helm-docs-template.gotmpl

helmchart/otel-add-on/templates/NOTES.txt

@@ -21,37 +21,61 @@
 {{- $collectors := include "operatorCrs" . | fromJsonArray }}
 Following components were installed:
 {{- if .Values.deployScaler }}
-- KEDA-OTel add-on ✓
+ ✓ KEDA-OTel add-on - 🐿
 {{- end }}
 {{- if .Values.otelOperator.enabled }}
-- OTel Operator ✓
+ ✓ OTel Operator - 🦑
 {{- end }}
 {{- range $collectors }}
-- OTel Collector with name '{{ .name | default $.Release.Name }}' (using the operator) ✓
+ ✓ OTel Collector with name '{{ .name | default $.Release.Name }}' (using the operator) -
+{{- if .targetAllocatorEnabled -}}
+{{- printf " target allocator mode watching Prom's CRs 🎯" -}}
+{{- else -}}
+{{- printf " 🚀" -}}
+{{- end -}}
 {{- end }}
 {{- if .Values.otelCollector.enabled }}
-- OTel Collector (using the helm chart) ✓
-{{- $collectors = append $collectors .Values.otelCollector -}}
+ ✓ OTel Collector (using the helm chart) - 👷
+{{- $collectors = append $collectors (mergeOverwrite .Values.otelCollector (dict "helm" true)) -}}
 {{- end }}
 
 Architecture:
+{{- $char := "" -}}
+{{- $m := "" -}}
+{{- if .Values.otelOperator.enabled }}
+                   ┏━━━━┓
+                   ┃ 🦑 ┃
+                  ╭╄┯┯┯┯╃╮
+
+{{- end }}
 {{- printf "\n" -}}
 {{- if $collectors }}
 {{- range $collectors }}
 {{- printf "   ┌──────────────┐    " -}}
 {{- end }}
 {{- printf "\n" -}}
 {{- range $collectors }}
-{{- printf "   │      🚀      │    " -}}
+{{- $char = "🚀" -}}
+{{- if .helm -}}
+{{- $char = "👷" -}}
+{{- end -}}
+{{- if .targetAllocatorEnabled -}}
+{{- $char = "🎯" -}}
+{{- end -}}
+{{- printf "   │      %s      │    " $char -}}
 {{- end }}
 {{- printf "\n" -}}
 {{- range $collectors }}
-{{- printf "   │ %-13s│    " ((.name | default "byHelmChart") | trunc 11) -}}
+{{- printf "   │ %-13s│    " ((.name | default $.Release.Name) | trunc 11) -}}
 {{- end }}
 {{- printf "  (Collectors)" -}}
 {{- printf "\n" -}}
 {{- range $collectors }}
-{{- printf "   │ %-13s│    " (printf "(%s)" (.mode | trunc 11)) -}}
+{{- $m = .mode -}}
+{{- if and .targetAllocatorEnabled (eq .mode "sidecar") -}}
+{{- $m = "statefulset" -}}
+{{- end -}}
+{{- printf "   │ %-13s│    " (printf "•%s" ($m | trunc 11)) -}}
 {{- end }}
 {{- printf "\n" -}}
 {{- range $collectors }}
@@ -72,12 +96,15 @@ Architecture:
 {{- if and .Values.deployScaler  $collectors }}
 {{- printf " ╵ \n ╵ \n ╵   " -}}
 {{- end }}
+{{- else }}
+(No collectors yet)
+
 {{- end }}
 
 {{- if .Values.deployScaler }}
-{{- printf "  ╭──────────────────╮         ╭╸╸╸╸╸╸╮\n" -}}
+{{- printf "  ╭─────────────────────╮         ╭╶╶╶╶╶╶╮\n" -}}
 {{- if $collectors -}}{{- printf " └╴╴→" -}}{{- end -}}
-{{- printf "  │ KEDA OTel Scaler │ ←╴╴╴╴╴→ ┆ KEDA ┆\n" -}}
+{{- printf "  │ KEDA OTel Scaler 🐿  │ ←╴╴╴╴╴→ │ KEDA │\n" -}}
 {{- if $collectors -}}{{- printf "     " -}}{{- end -}}
-{{- printf "  ╰──────────────────╯         ╰╸╸╸╸╸╸╯\n" -}}
+{{- printf "  ╰─────────────────────╯         ╰╶╶╶╶╶╶╯\n" -}}
 {{- end }}

helmchart/otel-add-on/templates/_helpers.tpl

@@ -70,7 +70,7 @@ Merge the default settings
         {{- $merged := dict }}
         {{- range .Values.otelOperatorCrs }}
           {{- if .enabled }}
-          {{- $merged = merge . $.Values.otelOperatorCrDefaultTemplate }}
+          {{- $merged = mergeOverwrite $.Values.otelOperatorCrDefaultTemplate . }}
           {{- $allmerged = append $allmerged $merged }}
           {{- end }}
         {{- end }}

helmchart/otel-add-on/templates/install-otc/otc-additional-rbac.yaml

@@ -69,7 +69,7 @@ metadata:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
-  name: {{ . }}
+  name: {{ . }}-otc
   labels:
     {{- include "otel-add-on.labels" $ | nindent 4 }}
 roleRef:

helmchart/otel-add-on/templates/install-otc/otc-configmap.yaml

@@ -15,12 +15,22 @@ data:
     kind: OpenTelemetryCollector
     metadata:
       name: {{ .name | default $.Release.Name }}
+      {{- if .targetAllocatorEnabled }}
+      namespace: {{ default $.Release.Namespace }}
+      {{- else }}
       namespace: {{ .namespace | default $.Release.Namespace }}
-      annotations:
+      {{- end }}
+      labels:
+        meta.helm.sh/release-name: {{ $.Release.Name }}
+        helm.sh/chart: {{ $.Chart.Version }}
     spec:
       resources:
         {{- toYaml .resources | nindent 8 }}
-      mode:  {{ .mode }}
+      {{- if and .targetAllocatorEnabled (eq .mode "sidecar") }}
+      mode: statefulset
+      {{- else }}
+      mode: {{ .mode }}
+      {{- end}}
       {{- with .serviceAccount.name }}
       serviceAccount: {{ . }}
       {{- end }}
@@ -63,12 +73,27 @@ data:
             {{- toYaml $receiverCfg | nindent 12 }}
         {{- end }}
         {{- else}}
-        {{- with .prometheusScrapeConfigs}}
-          {{- $receivers = append $receivers "prometheus" }}
+        {{- if or .targetAllocatorEnabled .prometheusScrapeConfigs }}
           prometheus:
             config:
               scrape_configs:
-                {{- toYaml . | nindent 16 }}
+        {{- $receivers = append $receivers "prometheus" }}
+        {{- if .targetAllocatorEnabled }}
+              - job_name: 'otel-collector'
+                scrape_interval: 10s
+                static_configs:
+                - targets: [ '0.0.0.0:8888' ]
+                metric_relabel_configs:
+                - action: labeldrop
+                  regex: (id|name)
+                - action: labelmap
+                  regex: label_(.+)
+                  replacement: $$1
+        {{- else }}
+        {{- with .prometheusScrapeConfigs }}
+              {{- toYaml . | nindent 14 }}
+        {{- end }}
+        {{- end }}
         {{- end }}
         {{- end }}
         {{- $processors := list }}

helmchart/otel-add-on/templates/install-otc/otc-delete-job.yaml

@@ -0,0 +1,72 @@
+{{- if (include "atLeastOneOperatorCr" $) -}}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: delete-otc
+  namespace: {{ default .Release.Namespace }}
+  labels:
+    app: delete-otc
+  annotations:
+    helm.sh/hook: pre-delete
+    helm.sh/hook-weight: "-1"
+    helm.sh/hook-delete-policy: "before-hook-creation,hook-succeeded"
+spec:
+  ttlSecondsAfterFinished: 43200 # 12h
+  backoffLimit: 4
+  template:
+    metadata:
+      annotations:
+        sidecar.istio.io/inject: "false"
+      labels:
+        app: delete-otc
+    spec:
+      restartPolicy: Never
+      serviceAccountName: manipulate-otc
+      securityContext:
+        runAsUser: 1000
+        runAsGroup: 2000
+      {{- with .Values.kubectlImage.pullSecrets }}
+      imagePullSecrets:
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        effect: NoSchedule
+      initContainers:
+      - name: wait-crds
+        image: "{{ .Values.kubectlImage.repository }}:{{ .Values.kubectlImage.tag }}"
+        imagePullPolicy: {{ .Values.kubectlImage.pullPolicy }}
+        command:
+        - sh
+        args:
+        - -c
+        - |
+          set -o nounset
+          for i in $(seq 20)
+          do
+            kubectl get crd opentelemetrycollectors.opentelemetry.io 2>&1 && exit 0
+            _sec=$(echo "1.5^$i" | bc)
+            echo "Waiting ${_sec} seconds for CRD opentelemetrycollectors.opentelemetry.io to exist"
+            sleep ${_sec}
+          done
+      containers:
+      - name: delete-otc
+        image: "{{ .Values.kubectlImage.repository }}:{{ .Values.kubectlImage.tag }}"
+        imagePullPolicy: {{ .Values.kubectlImage.pullPolicy }}
+        securityContext:
+          readOnlyRootFilesystem: true
+        command:
+        - sh
+        args:
+        - -c
+        - |
+          set -o nounset
+          for i in $(seq 20)
+          do
+            kubectl delete otelcol -A -lmeta.helm.sh/release-name={{.Release.Name}},helm.sh/chart={{.Chart.Version}} 2>&1 && exit 0
+            _sec=$(echo "1.5^$i" | bc)
+            echo "Waiting ${_sec} seconds.."
+            sleep ${_sec}
+          done
+          exit 1
+{{- end }}

…dd-on/templates/install-otc/otc-job.yaml …mplates/install-otc/otc-install-job.yamlhelmchart/otel-add-on/templates/install-otc/otc-job.yaml renamed to helmchart/otel-add-on/templates/install-otc/otc-install-job.yaml helmchart/otel-add-on/templates/install-otc/otc-job.yaml renamed to helmchart/otel-add-on/templates/install-otc/otc-install-job.yaml

@@ -21,7 +21,7 @@ spec:
         app: create-otc
     spec:
       restartPolicy: Never
-      serviceAccountName: install-otc
+      serviceAccountName: manipulate-otc
       securityContext:
         runAsUser: 1000
         runAsGroup: 2000

helmchart/otel-add-on/templates/install-otc/otc-job-rbac.yaml

@@ -3,19 +3,19 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: install-otc
+  name: manipulate-otc
   namespace: {{ .Release.Namespace }}
   annotations:
-    helm.sh/hook: "post-install,post-upgrade"
+    helm.sh/hook: "post-install,post-upgrade,pre-delete"
     helm.sh/hook-delete-policy: "before-hook-creation,hook-succeeded"
     helm.sh/hook-weight: "-2"
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  name: install-otc
+  name: manipulate-otc
   annotations:
-    helm.sh/hook: "post-install,post-upgrade"
+    helm.sh/hook: "post-install,post-upgrade,pre-delete"
     helm.sh/hook-delete-policy: "before-hook-creation,hook-succeeded"
     helm.sh/hook-weight: "-2"
 rules:
@@ -24,9 +24,7 @@ rules:
   resources:
   - opentelemetrycollectors
   verbs:
-  - patch
-  - create
-  - get
+  - "*"
 - apiGroups:
   - apiextensions.k8s.io
   resources:
@@ -37,17 +35,17 @@ rules:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
-  name: install-otc
+  name: manipulate-otc
   annotations:
-    helm.sh/hook: "post-install,post-upgrade"
+    helm.sh/hook: "post-install,post-upgrade,pre-delete"
     helm.sh/hook-delete-policy: "before-hook-creation,hook-succeeded"
     helm.sh/hook-weight: "-2"
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: install-otc
+  name: manipulate-otc
 subjects:
 - kind: ServiceAccount
-  name: install-otc
+  name: manipulate-otc
   namespace: {{ .Release.Namespace }}
 {{- end }}

helmchart/otel-add-on/values.yaml

@@ -1,7 +1,7 @@
 #otel-add-on:
 #------------
 replicaCount: 1
-namespace: keda
+namespace: ""
 ## -- when rendering/installing/upgrading the helm chart, the schema validation trivially passes
 #disableSchemaValidation: false
 
@@ -167,6 +167,7 @@ otelOperatorCrDefaultTemplate:
     - kube-prometheus-stack-prometheus
   targetAllocator:
     serviceAccount: otel-prom-reader
+    enabled: true
     prometheusCR:
       enabled: true
       # -- further narrow the ServiceMonitor CRs
@@ -175,6 +176,8 @@ otelOperatorCrDefaultTemplate:
       #          foo: bar
       # -- further narrow the ServiceMonitor CRs
       podMonitorSelector: {}
+      allowNamespaces: []
+      denyNamespaces: []
   serviceAccount:
     create: false
     annotations: {}
@@ -268,10 +271,8 @@ otelOperator:
     collectorImage:
 #      repository: otel/opentelemetry-collector-k8s
       repository: otel/opentelemetry-collector-contrib
-    env: {}
-    extraEnvs:
-    - name: ENABLE_WEBHOOKS
-      value: "false"
+    env:
+      ENABLE_WEBHOOKS: "false"
     serviceAccount:
       name: otel-operator
 
@@ -288,7 +289,6 @@ otelCollector:
   image:
     repository: otel/opentelemetry-collector-k8s
   fullnameOverride: otelcol
-
   ports:
     opencensus:
       enabled: true