Target allocator

charts/orchestrator-observability/deployments/orchestrator-observability/Chart.yaml

@@ -15,10 +15,9 @@ dependencies:
     condition: import.loki.enabled
     version: "~6.25.0"
     repository: "https://grafana.github.io/helm-charts"
-  - name: opentelemetry-collector
-    alias: opentelemetry-collector-daemonset
-    condition: import.otel.enabled
-    version: "~0.114.0"
+  - name: opentelemetry-operator
+    condition: import.otel-operator.enabled
+    version: "~0.86.1"
     repository: "https://open-telemetry.github.io/opentelemetry-helm-charts"
   - name: opentelemetry-collector
     condition: import.otel.enabled

charts/orchestrator-observability/deployments/orchestrator-observability/templates/opentelemetry-collector-cr.yaml

@@ -0,0 +1,195 @@
+{{- if .Values.opentelemetryCollectorCR.enabled }}
+apiVersion: opentelemetry.io/v1beta1
+kind: OpenTelemetryCollector
+metadata:
+  name: orchestrator-observability-otel
+spec:
+  mode: daemonset
+  upgradeStrategy: automatic
+  targetAllocator:
+    enabled: true
+    serviceAccount: opentelemetry-targetallocator-role
+    prometheusCR:
+      enabled: true
+      podMonitorSelector: {}
+      serviceMonitorSelector: {}
+  config:
+    extensions:
+      health_check:
+        endpoint: "0.0.0.0:13133"
+
+    receivers:
+      filelog:
+        exclude:
+          - /var/log/pods/orch-platform_orchestrator-observability-opentelemetry-collector-daemonset*_*/opentelemetry-collector-daemonset/*.log
+        include:
+          - /var/log/pods/*/*/*.log
+        include_file_name: false
+        include_file_path: true
+        operators:
+          - id: get-format
+            routes:
+              - expr: body matches "^\\{"
+                output: parser-docker
+              - expr: body matches "^[^ Z]+ "
+                output: parser-crio
+              - expr: body matches "^[^ Z]+Z"
+                output: parser-containerd
+            type: router
+          - id: parser-crio
+            regex: ^(?P<time>[^ Z]+) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) ?(?P<log>.*)$
+            timestamp:
+              layout: 2006-01-02T15:04:05.999999999Z07:00
+              layout_type: gotime
+              parse_from: attributes.time
+            type: regex_parser
+          - combine_field: attributes.log
+            combine_with: ""
+            id: crio-recombine
+            is_last_entry: attributes.logtag == 'F'
+            max_log_size: 102400
+            output: extract_metadata_from_filepath
+            source_identifier: attributes["log.file.path"]
+            type: recombine
+          - id: parser-containerd
+            regex: ^(?P<time>[^ ^Z]+Z) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) ?(?P<log>.*)$
+            timestamp:
+              layout: '%Y-%m-%dT%H:%M:%S.%LZ'
+              parse_from: attributes.time
+            type: regex_parser
+          - combine_field: attributes.log
+            combine_with: ""
+            id: containerd-recombine
+            is_last_entry: attributes.logtag == 'F'
+            max_log_size: 102400
+            output: extract_metadata_from_filepath
+            source_identifier: attributes["log.file.path"]
+            type: recombine
+          - id: parser-docker
+            output: extract_metadata_from_filepath
+            timestamp:
+              layout: '%Y-%m-%dT%H:%M:%S.%LZ'
+              parse_from: attributes.time
+            type: json_parser
+          - id: extract_metadata_from_filepath
+            parse_from: attributes["log.file.path"]
+            regex: ^.*\/(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[a-f0-9\-]+)\/(?P<container_name>[^\._]+)\/(?P<restart_count>\d+)\.log$
+            type: regex_parser
+          - from: attributes.stream
+            to: attributes["log.iostream"]
+            type: move
+          - from: attributes.container_name
+            to: resource["k8s.container.name"]
+            type: move
+          - from: attributes.namespace
+            to: resource["k8s.namespace.name"]
+            type: move
+          - from: attributes.pod_name
+            to: resource["k8s.pod.name"]
+            type: move
+          - from: attributes.log
+            to: body
+            type: move
+
+      kubeletstats:
+        collection_interval: 60s
+        insecure_skip_verify: true
+        extra_metadata_labels:
+          - k8s.volume.type
+        metric_groups:
+          - container
+          - pod
+          - volume
+          - node
+
+      prometheus:
+        config:
+          scrape_configs:
+          - job_name: 'otel-collector'
+            scrape_interval: 60s
+            static_configs:
+            - targets: [ '0.0.0.0:8888' ]
+
+    processors:
+      memory_limiter:
+        limit_percentage: 70
+        check_interval: 15s
+
+      resource/remove_container_id:
+        attributes:
+          - action: delete
+            key: container.id
+          - action: delete
+            key: container_id
+
+      resource:
+        attributes:
+          - action: insert
+            key: loki.resource.labels
+            value: k8s.container.name, k8s.namespace.name, k8s.pod.name
+          - action: insert
+            key: loki.format
+            value: raw
+
+      attributes:
+        actions:
+          - action: insert
+            key: otelcol_source
+            value: "platform-daemonset"
+
+      transform:
+        metric_statements:
+          - context: datapoint
+            statements:
+              - set(attributes["k8s_namespace_name"], resource.attributes["k8s.namespace.name"])
+              - set(attributes["k8s_container_name"], resource.attributes["k8s.container.name"])
+              - set(attributes["k8s_daemonset_name"], resource.attributes["k8s.daemonset.name"])
+              - set(attributes["k8s_deployment_name"], resource.attributes["k8s.deployment.name"])
+              - set(attributes["k8s_node_name"], resource.attributes["k8s.node.name"])
+              - set(attributes["k8s_pod_name"], resource.attributes["k8s.pod.name"])
+              - set(attributes["k8s_statefulset_name"], resource.attributes["k8s.statefulset.name"])
+
+    exporters:
+      otlphttp/logs:
+        endpoint: {{ .Values.opentelemetryCollectorCR.loki.endpoint }}
+        headers:
+          "X-Scope-OrgID": {{ .Values.opentelemetryCollectorCR.loki.orgId }}
+
+      prometheusremotewrite:
+        endpoint: {{ .Values.opentelemetryCollectorCR.mimir.endpoint }}
+        headers:
+          "X-Scope-OrgID": {{ .Values.opentelemetryCollectorCR.mimir.orgId }}
+        resource_to_telemetry_conversion:
+          enabled: true
+
+    service:
+      telemetry:
+        metrics:
+          level: "basic"
+          readers:
+            - pull:
+                exporter:
+                  prometheus:
+                    host: "0.0.0.0"
+                    port: 8888
+      extensions:
+        - health_check
+      pipelines:
+        logs:
+          receivers: [filelog]
+          processors:
+            - memory_limiter
+            - resource/remove_container_id
+            - resource
+          exporters:
+            - otlphttp/logs
+        metrics:
+          receivers: [prometheus, kubeletstats]
+          processors:
+            - memory_limiter
+            - attributes
+            - transform
+            - resource
+          exporters:
+            - prometheusremotewrite
+{{- end }}

charts/orchestrator-observability/deployments/orchestrator-observability/templates/opentelemetry-operator-clusterrole.yaml

@@ -0,0 +1,55 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: opentelemetry-targetallocator-role
+rules:
+  - apiGroups:
+      - monitoring.coreos.com
+    resources:
+      - servicemonitors
+      - podmonitors
+    verbs:
+      - '*'
+  - apiGroups: ['']
+    resources:
+      - namespaces
+    verbs: ['get', 'list', 'watch']
+  - apiGroups: ['']
+    resources:
+      - nodes
+      - nodes/metrics
+      - nodes/proxy
+      - nodes/stats
+      - services
+      - endpoints
+      - pods
+    verbs: ['get', 'list', 'watch']
+  - apiGroups: ['']
+    resources:
+      - configmaps
+    verbs: ['get']
+  - apiGroups:
+      - discovery.k8s.io
+    resources:
+      - endpointslices
+    verbs: ['get', 'list', 'watch']
+  - apiGroups:
+      - networking.k8s.io
+    resources:
+      - ingresses
+    verbs: ['get', 'list', 'watch']
+  - nonResourceURLs: ['/metrics']
+    verbs: ['get']
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: opentelemetry-targetallocator-rb
+subjects:
+  - kind: ServiceAccount
+    name: opentelemetry-operator
+    namespace: opentelemetry
+roleRef:
+  kind: ClusterRole
+  name: opentelemetry-targetallocator-role
+  apiGroup: rbac.authorization.k8s.io