Logging POC initial commit

Author: ilanRosenbaum

logging/es-config.yaml

@@ -0,0 +1,92 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: elasticsearch
+  labels:
+    app: elasticsearch
+spec:
+  selector:
+    app: elasticsearch
+  clusterIP: None
+  ports:
+    - port: 9200
+      name: rest
+    - port: 9300
+      name: inter-node
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: es-cluster
+spec:
+  serviceName: elasticsearch
+  replicas: 3
+  selector:
+    matchLabels:
+      app: elasticsearch
+  template:
+    metadata:
+      labels:
+        app: elasticsearch
+    spec:
+      containers:
+      - name: elasticsearch
+        image: docker.elastic.co/elasticsearch/elasticsearch:7.5.0
+        resources:
+            limits:
+              cpu: 1000m
+            requests:
+              cpu: 100m
+        ports:
+        - containerPort: 9200
+          name: rest
+          protocol: TCP
+        - containerPort: 9300
+          name: inter-node
+          protocol: TCP
+        volumeMounts:
+        - name: data
+          mountPath: /usr/share/elasticsearch/data
+        env:
+          - name: cluster.name
+            value: k8s-logs
+          - name: node.name
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: discovery.seed_hosts
+            value: "es-cluster-0.elasticsearch,es-cluster-1.elasticsearch,es-cluster-2.elasticsearch"
+          - name: cluster.initial_master_nodes
+            value: "es-cluster-0,es-cluster-1,es-cluster-2"
+          - name: ES_JAVA_OPTS
+            value: "-Xms512m -Xmx512m"
+      initContainers:
+      - name: fix-permissions
+        image: busybox
+        command: ["sh", "-c", "chown -R 1000:1000 /usr/share/elasticsearch/data"]
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: data
+          mountPath: /usr/share/elasticsearch/data
+      - name: increase-vm-max-map
+        image: busybox
+        command: ["sysctl", "-w", "vm.max_map_count=262144"]
+        securityContext:
+          privileged: true
+      - name: increase-fd-ulimit
+        image: busybox
+        command: ["sh", "-c", "ulimit -n 65536"]
+        securityContext:
+          privileged: true
+  volumeClaimTemplates:
+  - metadata:
+      name: data
+      labels:
+        app: elasticsearch
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+      # storageClassName: ""
+      resources:
+        requests:
+          storage: 3Gi

logging/fd-configmap.yaml

@@ -0,0 +1,54 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: fluentd-config
+  labels:
+    kubernetes.io/cluster-service: "true"
+data:
+  fluent.conf: |
+      <match fluent.**>
+        @type null
+      </match>
+
+      <source>
+        @type tail
+        @id tail_marklogic_logs
+        format none
+        path /var/opt/MarkLogic/Logs/*
+        pos_file /tmp/log/ml-logs.log.pos
+        path_key log_file
+        tag marklogic.logs
+        read_from_head true
+        <parse>
+          @type none
+          time_key @timestamp
+          time_format %Y-%m-%dT%H:%M:%S.%N%z
+          keep_time_key true
+        </parse>
+      </source>
+
+      <filter marklogic.logs>
+        @type record_transformer
+        enable_ruby true
+        <record>
+          # overwrite full path to file name
+          log_file ${File.basename(record["log_file"])}
+        </record>
+      </filter>
+
+      <match marklogic.logs>
+        @type elasticsearch
+        host elasticsearch.default.svc.cluster.local
+        port 9200
+        user admin
+        password admin
+        index_name fluentd.${tag}
+        include_timestamp true
+        utc_index false
+        time_key_format "%Y-%m-%dT%H:%M:%S.%N%z"
+        time_key time
+        logstash_format true
+        logstash_prefix marklogic.logs
+        path_key out/${log_file}
+        flush_interval 1s
+      </match>

logging/kibana-config.yaml

@@ -0,0 +1,41 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: kibana
+  labels:
+    app: kibana
+spec:
+  ports:
+  - port: 5601
+  selector:
+    app: kibana
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kibana
+  labels:
+    app: kibana
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: kibana
+  template:
+    metadata:
+      labels:
+        app: kibana
+    spec:
+      containers:
+      - name: kibana
+        image: docker.elastic.co/kibana/kibana:7.5.0
+        resources:
+          limits:
+            cpu: 1000m
+          requests:
+            cpu: 100m
+        env:
+          - name: ELASTICSEARCH_URL
+            value: http://elasticsearch:9200
+        ports:
+        - containerPort: 5601

logging/reset.sh

@@ -0,0 +1,11 @@
+minikube delete                                                         
+minikube start --driver=docker -n=1
+
+kubectl apply -f logging/es-config.yaml
+kubectl apply -f logging/kibana-config.yaml
+kubectl port-forward kibana-7f4cc47bkw4d-db7dn 5601:5601
+
+echo "=====Loading marklogc images to minikube cluster"
+minikube image load marklogic-centos/marklogic-server-centos:10-internal
+
+helm install ml ./charts/ -f values.yaml # Use local charts for helm install