Ekscplogs (#50)

* EKS control plane logs collection configuration

---------
Co-authored-by: paliwalparitosh <[email protected]>

Author: pankajtalk

CHANGELOG.md

@@ -1,13 +1,16 @@
 # Change Log
 
+## 2024-03-08
+### Added
+- Support for AWS EKS system and control plane logs collection.
+
 ## 2024-02-13
 ### Added
 - Changes to support Kubernetes Solution Pages Offering by OCI Logging Analytics.
   - A new role and role binding in the monitoring namespace (which defaults to oci-onm), to manage a ConfigMap.
   - A new CronJob to handle the Kubernetes Objects discovery and Objects Logs collection using oci-logging-analytics-kubernetes-discovery Gem.
 ### Changed
 - Moving forward, Kubernetes Objects logs would be collected using Kubernetes Discovery CronJob along with the (optional) Discovery data instead of Fluentd based Deployment.
-
 ## 2024-01-18
 ### Changed
 - Management Agent docker image has been updated to version 1.2.0

charts/logan/Chart.yaml

@@ -5,7 +5,7 @@ apiVersion: v2
 name: oci-onm-logan
 description: Charts for sending Kubernetes platform logs, compute logs, and Kubernetes Objects information to OCI Logging Analytics.
 type: application
-version: 3.3.0
+version: 3.4.0
 appVersion: "3.0.0"
 
 dependencies:

charts/logan/templates/ekscp-logs-configmap.yaml

@@ -0,0 +1,106 @@
+# Copyright (c) 2023, Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v1.0 as shown at https://oss.oracle.com/licenses/upl.
+
+---
+{{- if .Values.enableEKSControlPlaneLogs }}
+{{- $authtype := .Values.authtype | lower }}
+{{- $imagePullSecrets := .Values.image.imagePullSecrets }}
+{{- $resourceNamePrefix := (include "logan.resourceNamePrefix" .) }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ $resourceNamePrefix }}-logan
+  namespace: {{ include "logan.namespace" . }}
+  labels:
+    app: {{ $resourceNamePrefix }}-logan
+    version: v1
+spec:
+  selector:
+    matchLabels:
+      app: {{ $resourceNamePrefix }}-logan
+      version: v1
+  template:
+    metadata:
+      annotations:
+        {{- if eq $authtype "config" }}
+        checksum/secrets: {{ include (print $.Template.BasePath "/oci-config-secret.yaml") . | sha256sum }}
+        {{- end}}
+        checksum/ekscpconfigmap: {{ include (print $.Template.BasePath "/ekscp-logs-configmap.yaml") . | sha256sum }}
+      labels:
+        app: {{ $resourceNamePrefix }}-logan
+        version: v1
+    spec:
+      serviceAccountName: {{ include "logan.serviceAccount" . }}
+      {{- if $imagePullSecrets }}
+      imagePullSecrets:
+      - name: {{ .Values.image.imagePullSecrets }}
+      {{- end}}
+      containers:
+      - name: {{ $resourceNamePrefix }}-ekscp-fluentd
+        image: {{ .Values.image.url }}
+        imagePullPolicy: {{ default "IfNotPresent" .Values.image.imagePullPolicy }}
+        env:
+          - name: FLUENTD_CONF
+            value: {{ .Values.fluentd.path }}/{{ .Values.fluentd.file }}
+          - name: K8S_NODE_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.nodeName
+          - name: FLUENT_OCI_DEFAULT_LOGGROUP_ID
+            value: {{ .Values.ociLALogGroupID }}
+          - name: FLUENT_OCI_NAMESPACE
+            value: {{ .Values.ociLANamespace }}
+          - name: FLUENT_OCI_KUBERNETES_CLUSTER_ID
+            value: {{ include "logan.kubernetesClusterId" . }}
+          - name: FLUENT_OCI_KUBERNETES_CLUSTER_NAME
+            value: {{ include "logan.kubernetesClusterName" . }}
+          {{- if eq $authtype "config" }}
+          - name: FLUENT_OCI_CONFIG_LOCATION
+            value: {{ .Values.oci.path }}/{{ .Values.oci.file }}
+          {{- end }}
+          {{- if .Values.extraEnv }}
+          {{- toYaml .Values.extraEnv | nindent 10 }}
+          {{- end }}
+        {{- if .Values.resources }}
+        resources: {{- toYaml .Values.resources | nindent 10 }}
+        {{- end }}
+        volumeMounts:
+        # RW mount to store tail plugin output plugin buffer and logs
+        - name: basedir
+          mountPath: {{ .Values.fluentd.baseDir }}
+        {{- if eq $authtype "config" }}
+        # Mount directory where oci config exists
+        - name: ociconfigdir
+          mountPath: {{ .Values.oci.path }}
+          readOnly: true
+        {{- end }}
+        # Mount directory where fluentd config exists
+        - name: ekscpfluentdconfigdir
+          mountPath: {{ .Values.fluentd.path }}
+          readOnly: true
+        {{- if .Values.extraVolumeMounts }}
+        {{- toYaml .Values.extraVolumeMounts | nindent 8 }}
+        {{- end }}
+      terminationGracePeriodSeconds: 30
+      volumes:
+      {{- if .Values.extraVolumes }}
+      {{- toYaml .Values.extraVolumes | nindent 6 }}
+      {{- end }}
+      # RW mount to store tail plugin output plugin buffer and logs
+      - name: basedir
+        hostPath:
+          path: {{ .Values.fluentd.baseDir }}
+      {{- if eq $authtype "config" }}
+      # Mount directory where oci config exists
+      - name: ociconfigdir
+        projected:
+          sources:
+              - secret:
+                  name: {{ $resourceNamePrefix }}-oci-config
+      {{- end }}
+      # Mount directory where fluentd ekscp config exists
+      - name: ekscpfluentdconfigdir
+        configMap:
+          # Provide the name of the ConfigMap to mount.
+          name: {{ $resourceNamePrefix }}-ekscp-logs
+{{- end }}

charts/logan/templates/fluentd-deployment.yaml

@@ -52,6 +52,18 @@
           "type": "string"
         }
       }
+    },
+    "collectionType": {
+      "type": "string",
+      "enum": ["cloudwatch", "s3"]
+    },
+    "region": {
+      "type": "string"
+    },
+    "s3Bucket": {
+      "type": "string",
+      "minLength": 3,
+      "maxLength": 63
     }
   }
 }

charts/logan/values.schema.json

@@ -48,7 +48,7 @@ image:
    # Image pull secrets for. Secret must be in the namespace defined by namespace
    imagePullSecrets:
    # -- Replace this value with actual docker image url
-   url: container-registry.oracle.com/oci_observability_management/oci-la-fluentd-collector:1.3.0
+   url: container-registry.oracle.com/oci_observability_management/oci-la-fluentd-collector:1.4.0
    # -- Image pull policy
    imagePullPolicy: Always
 
@@ -60,7 +60,7 @@ ociLANamespace:
 # e.g. ocid1.loganalyticsloggroup.oc1.phx.amaaaaasdfaskriauucc55rlwlxe4ahe2vfmtuoqa6qsgu7mb6jugxacsk6a
 ociLALogGroupID:
 
-# -- OKE Cluster OCID
+# -- OKE Cluster OCID/EKS Cluster ARN etc.
 # e.g. ocid1.cluster.oc1.phx.aaaaaaaahhbadf3rxa62faaeixanvr7vftmkg6hupycbf4qszctf2wbmqqxq
 kubernetesClusterID:
 
@@ -77,6 +77,9 @@ ociLAClusterEntityID:
 # In Kubernetes environments where SELinux mode is enforced, set this flag to 'true' to allow fluentd pods to access log files.
 privileged: false
 
+# -- Enables collection of AWS EKS Control Plane logs through CloudWatch or S3 Fluentd plugin
+enableEKSControlPlaneLogs: false
+
 # Logging Analytics additional metadata. Use this to tag all the collected logs with one or more key:value pairs.
 # Key must be a valid field in Logging Analytics
 #metadata:
@@ -289,7 +292,7 @@ fluentd:
          # -- Kubernetes CSI Node Driver Logs collection configuration
          csinode:
             # csinode log files location.
-            path: /var/log/containers/csi-oci-node-*.log
+            path: /var/log/containers/csi-oci-node-*.log,/var/log/containers/ebs-csi-node-*.log
             # Logging Analytics log source to use for parsing and processing Kubernetes CSI Node Driver Logs.
             ociLALogSourceName: "Kubernetes CSI Node Driver Logs"
 
@@ -309,6 +312,13 @@ fluentd:
             #  The regular expression pattern for the starting line in case of multi-line logs.
             multilineStartRegExp: /^\S\d{2}\d{2}\s+[^\:]+:[^\:]+:[^\.]+\.\d{0,3}/
 
+         # -- Kubernetes CSI Controller Logs collection configuration
+         csi-controller:
+            # csi controller log files location.
+            path: /var/log/containers/ebs-csi-controller-*.log
+            # Logging Analytics log source to use for parsing and processing Kubernetes CSI Controller Logs.
+            ociLALogSourceName: "Kubernetes CSI Controller Logs"
+
          # Config specific to API Server Logs Collection
          kube-apiserver:
             # The path to the source files.
@@ -425,6 +435,122 @@ fluentd:
             # Logging Analytics log source to use for parsing and processing Linux YUM Logs.
             ociLALogSourceName: "Linux YUM Logs"
 
+   # Configuration for AWS EKS Control Plane logs like API Server, Audit, Authenticator etc.
+   eksControlPlane:
+      # Collection Type (cloudwatch or s3)
+      collectionType: "cloudwatch"
+      # AWS region
+      region:
+      # Use AssumeRoleCredentials (https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/AssumeRoleCredentials.html) to authenticate
+      # Default is true. Set to false to use access keys
+      awsUseSts: true
+      # The role ARN to assume when using AWS Security Token Service authentication
+      awsStsRoleArn:
+      # AWS access key and secret access key, needed only when awsUseSts is explicitly set to false
+      #awsKeyId:
+      #awsSecKey:
+      # CloudWatch Log Group name of the EKS cluster. Automatically determined by extracting cluster name from kubernetesClusterId and
+      # following naming syntax as "aws/eks/<cluster name>/cluster". Below field can be used to override this behavior.
+      #cwLogGroupName:
+      # S3 related settings
+      # S3 bucket name to which EKS Control Plane logs are being streamed using a subscription filter
+      s3Bucket:
+      ociLALogGroupID:
+      #metadata:
+         #"Client Host Region": "America"
+         #"Environment": "Production"
+         #"Third Key": "Third Value"
+      # Worker number in case of multi process workers enabled. If not set when multi process workers enabled, then it defaults to 0.
+      #worker:
+      logs:
+         # If using cloudwatch collection mechanism, apiserver and audit logs need to be part of the same worker as they share the same log stream name prefix.
+         # Thus "worker" variable is only picked up from "apiserver" section.
+         apiserver:
+            # CloudWatch Log Stream name
+            cwLogStreamName: "kube-apiserver"
+            # SQS queue name which is notified when apiserver log object is created in S3 bucket
+            sqsQueue: "apiserver"
+            # S3 object key
+            objectKey: .*?kube-apiserver/
+            # Logging Analytics log source to use for parsing and processing EKS Control Plane API Server Logs.
+            ociLALogSourceName: "Kubernetes API Server Logs"
+            multilineStartRegExp: /^\S\d{2}\d{2}\s+[^\:]+:[^\:]+:[^\.]+\.\d{0,3}/
+            #metadata:
+               #"Client Host Region": "America"
+               #"Environment": "Production"
+               #"Third Key": "Third Value"
+            #ociLALogGroupID:
+            # Worker number in case of multi process workers enabled. If not set when multi process workers enabled, then it defaults to 0.
+            #worker:
+         audit:
+            sqsQueue: "audit"
+            # S3 object key
+            objectKey: .*?kube-apiserver-audit
+            # Logging Analytics log source to use for parsing and processing EKS Control Plane Audit Logs.
+            ociLALogSourceName: "Kubernetes Audit Logs"
+            #metadata:
+               #"Client Host Region": "America"
+               #"Environment": "Production"
+               #"Third Key": "Third Value"
+            #ociLALogGroupID:
+            #worker:
+         authenticator:
+            cwLogStreamName: "authenticator"
+            sqsQueue: "authenticator"
+            # S3 object key
+            objectKey: .*?authenticator
+            # Logging Analytics log source to use for parsing and processing EKS Control Plane Authenticator Logs.
+            ociLALogSourceName: "AWS EKS Authenticator Logs"
+            multilineStartRegExp: /^time=/
+            #metadata:
+               #"Client Host Region": "America"
+               #"Environment": "Production"
+               #"Third Key": "Third Value"
+            #ociLALogGroupID:
+            #worker:
+         kubecontrollermanager:
+            cwLogStreamName: "kube-controller-manager"
+            sqsQueue: "kube-controller-manager"
+            # S3 object key
+            objectKey: .*?kube-controller-manager
+            # Logging Analytics log source to use for parsing and processing EKS Control Plane Kube Controller Manager Logs.
+            ociLALogSourceName: "Kubernetes Controller Manager Logs"
+            multilineStartRegExp: /^\S\d{2}\d{2}\s+[^\:]+:[^\:]+:[^\.]+\.\d{0,3}/
+            #metadata:
+               #"Client Host Region": "America"
+               #"Environment": "Production"
+               #"Third Key": "Third Value"
+            #ociLALogGroupID:
+            #worker:
+         cloudcontrollermanager:
+            cwLogStreamName: "cloud-controller-manager"
+            sqsQueue: "cloud-controller-manager"
+            # S3 object key
+            objectKey: .*?cloud-controller-manager
+            # Logging Analytics log source to use for parsing and processing EKS Control Plane Cloud Controller Manager Logs.
+            ociLALogSourceName: "Cloud Controller Manager Logs"
+            multilineStartRegExp: /^\S\d{2}\d{2}\s+[^\:]+:[^\:]+:[^\.]+\.\d{0,3}/
+            #metadata:
+               #"Client Host Region": "America"
+               #"Environment": "Production"
+               #"Third Key": "Third Value"
+            #ociLALogGroupID:
+            #worker:
+         scheduler:
+            cwLogStreamName: "kube-scheduler"
+            sqsQueue: "scheduler"
+            # S3 object key
+            objectKey: .*?kube-scheduler
+            # Logging Analytics log source to use for parsing and processing EKS Control Plane Scheduler Logs.
+            ociLALogSourceName: "Kubernetes Scheduler Logs"
+            multilineStartRegExp: /^\S\d{2}\d{2}\s+[^\:]+:[^\:]+:[^\.]+\.\d{0,3}/
+            #metadata:
+               #"Client Host Region": "America"
+               #"Environment": "Production"
+               #"Third Key": "Third Value"
+            #ociLALogGroupID:
+            #worker:
+
    # Generic configuration for all container/pod logs
    genericContainerLogs:
       # -- Default Logging Analytics log source to use for parsing and processing the logs: Kubernetes Container Generic Logs.
@@ -440,6 +566,8 @@ fluentd:
          - '"/var/log/containers/csi-oci-node-*.log"'
          - '"/var/log/containers/proxymux-client-*.log"'
          - '"/var/log/containers/cluster-autoscaler-*.log"'
+         - '"/var/log/containers/ebs-csi-node-*.log"'
+         - '"/var/log/containers/ebs-csi-controller-*.log"'
          - '"/var/log/containers/kube-apiserver-*.log"'
          - '"/var/log/containers/etcd-*.log"'
          - '"/var/log/containers/kube-controller-manager-*.log"'

charts/logan/values.yaml

@@ -18,7 +18,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 3.3.0
+version: 3.4.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
@@ -32,7 +32,7 @@ dependencies:
   repository: "file://../common"
   condition: oci-onm-common.enabled
 - name: oci-onm-logan
-  version: "3.3.0"
+  version: "3.4.0"
   repository: "file://../logan"
   condition: oci-onm-logan.enabled
 - name: oci-onm-mgmt-agent

charts/oci-onm/Chart.yaml

@@ -31,7 +31,7 @@ oci-onm-logan:
   kubernetesClusterID: "{{ .Values.global.kubernetesClusterID }}"
   kubernetesClusterName: "{{ .Values.global.kubernetesClusterName }}"
   image:
-    url: container-registry.oracle.com/oci_observability_management/oci-la-fluentd-collector:1.3.0
+    url: container-registry.oracle.com/oci_observability_management/oci-la-fluentd-collector:1.4.0
   # Go to OCI Logging Analytics Administration, click Service Details, and note the namespace value.
   ociLANamespace:
   # OCI Logging Analytics Default Log Group OCID