Merge branch 'main' of https://github.com/osodevops/confluent-kubernetes-playground into format

Author: sionsmith

.gitignore

@@ -10,4 +10,13 @@
 .build-harness
 build-harness
 
-test.log
+sensitive-*
+identity
+identity.pub
+known_hosts
+ca.pem
+intermediate.cert.pem
+pki_intermediate.csr
+pki-ca-root.json
+cert_key_list
+nohup.out

examples/elk-logging/Dockerfile

@@ -0,0 +1,3 @@
+FROM docker.elastic.co/logstash/logstash:7.14.0
+#RUN bin/logstash-plugin install logstash-output-kafka
+RUN bin/logstash-plugin install microsoft-logstash-output-azure-loganalytics

examples/elk-logging/README.md

@@ -0,0 +1,59 @@
+# Monitoring (JMX/Prometheus/Grafana)
+In this example, we deploy an RBAC enabled Confluent cluster with Prometheus/Grafana integration.
+
+## Deploy Stack
+From within this present directory(./examples/monitoring), run the following command:
+
+```shell
+kubectl apply -k ../../kustomize/crds && sleep 1 && kubectl apply -k .
+```
+    
+Once all the the pods are in a 'Running' status, we can start to investigate the rest of the stack.
+
+## Prometheus
+Prometheus has a UI you can view by forwarding port 9090 with the following command, and then accessing `http://127.0.0.1:9090` from a local browser.
+```shell
+kubectl port-forward \
+$(kubectl get pods -n sandbox -l app=prometheus -l component=server -o name) \
+9090 --namespace sandbox
+```
+
+If you navigate to `http://localhost:9090/targets` you should hopefully see a screen such as this which indicates that Prometheus is successfully scraping from the Confluent Services
+![](../../resources/images/prometheus_targets.png)0
+
+The configuration file for Prometheus can be found at `./examples/monitoring/prometheus/server/cm.yaml` 
+
+## Grafana
+Grafana's Web UI runs on port 3000.  Similar to Prometheus, forward that port to your local machines with the following command:
+```shell
+kubectl port-forward \
+$(kubectl get pods -n sandbox kubectl get pods -n sandbox -l app.kubernetes.io/component=grafana -o name) \
+3000 --namespace sandbox
+```
+
+Navigate to http://localhost:3000
+
+You will need to login with the username 'admin' and the password 'password'.  At the home screen you will see a dashboard called 'Confluent Platform'.  Click on this
+![](../../resources/images/grafana-dashboard.png)
+
+This dashboard will display various metrics/alerts for the various Confluent services.  This dashboard has been captured in code, and can be edited at `./examples/grafana/dashboards.yaml`
+![](../../resources/images/grafana-dashboard2.png)
+
+
+
+
+```
+curl -X POST "http://localhost:5601/api/index_patterns/index_pattern" -H "Content-Type:application/json" -H 'kbn-xsrf:true' -d'{"index_pattern": {"title": "hello"}}'
+
+
+curl -X POST "http://localhost:5601/api/index_patterns/default" -H "Content-Type:application/json" -H 'kbn-xsrf:true' -d'{"index_pattern_id": "logstash*","force": true}'
+
+
+curl -X GET "http://localhost:5601/api/index_patterns/1" -H "Content-Type:application/json" -H 'kbn-xsrf:true'
+
+curl -X GET "http://localhost:5601/api/index_patterns/log*" -H "Content-Type:application/json" -H 'kbn-xsrf:true'
+curl -X GET "http://localhost:5601/api/index_patterns/log" -H "Content-Type:application/json" -H 'kbn-xsrf:true'
+
+
+curl -X GET "http://localhost:5601/api/index_patterns" -H "Content-Type:application/json" -H 'kbn-xsrf:true'
+```

examples/elk-logging/build-inside.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+eval $(minikube docker-env)
+docker build -t sandbox-custom-logstash .

examples/elk-logging/confluent/control-centre.yaml

@@ -0,0 +1,42 @@
+apiVersion: platform.confluent.io/v1beta1
+kind: ControlCenter
+metadata:
+  name: controlcenter
+spec:
+  authorization:
+    type: rbac
+  tls:
+    secretRef: tls-group1
+  dependencies:
+    kafka:
+      bootstrapEndpoint: kafka.sandbox.svc.cluster.local:9071
+      authentication:
+        type: plain
+        jaasConfig:
+          secretRef: mds-client-connect
+      tls:
+        enabled: true
+    mds:
+      endpoint: https://kafka.sandbox.svc.cluster.local:8090
+      tokenKeyPair:
+        secretRef: mds-public
+      authentication:
+        type: bearer
+        bearer:
+          secretRef: mds-client-c3
+      tls:
+        enabled: true
+    connect:
+      - name: connect
+        url:  https://connect.sandbox.svc.cluster.local:8083
+        tls:
+          enabled: true
+    ksqldb:
+      - name: ksqldb
+        url:  https://ksqldb.sandbox.svc.cluster.local:8088
+        tls:
+          enabled: true
+    schemaRegistry:
+      url: https://schemaregistry.sandbox.svc.cluster.local:8081
+      tls:
+        enabled: true

examples/elk-logging/confluent/kafka-connect.yaml

@@ -0,0 +1,27 @@
+---
+apiVersion: platform.confluent.io/v1beta1
+kind: Connect
+metadata:
+  name: connect
+spec:
+  tls:
+    secretRef: tls-group1
+  authorization:
+    type: rbac
+  dependencies:
+    kafka:
+      bootstrapEndpoint: kafka.sandbox.svc.cluster.local:9071
+      authentication:
+        type: plain
+        jaasConfig:
+          secretRef: mds-client-connect
+      tls:
+        enabled: true
+    mds:
+      endpoint: https://kafka.sandbox.svc.cluster.local:8090
+      tokenKeyPair:
+        secretRef: mds-public
+      authentication:
+        type: bearer
+        bearer:
+          secretRef: mds-client-connect

examples/elk-logging/confluent/kafka.yaml

@@ -0,0 +1,81 @@
+---
+apiVersion: platform.confluent.io/v1beta1
+kind: Kafka
+metadata:
+  name: kafka
+spec:
+  configOverrides:
+    server:
+      - confluent.schema.registry.url=https://schemaregistry.sandbox.svc.cluster.local:8081
+      - listener.name.internal.plain.sasl.server.callback.handler.class=io.confluent.security.auth.provider.ldap.LdapAuthenticateCallbackHandler
+      - listener.name.external.plain.sasl.server.callback.handler.class=io.confluent.security.auth.provider.ldap.LdapAuthenticateCallbackHandler
+      - listener.name.replication.plain.sasl.server.callback.handler.class=io.confluent.security.auth.provider.ldap.LdapAuthenticateCallbackHandler
+  tls:
+    secretRef: tls-group1
+  metricReporter:
+    enabled: true
+    authentication:
+      type: plain
+      jaasConfigPassThrough:
+        secretRef: broker-credential
+    tls:
+      enabled: true
+  listeners:
+    internal:
+      authentication:
+        type: plain
+        jaasConfigPassThrough:
+          secretRef: broker-credential
+      tls:
+        enabled: true
+    external:
+      authentication:
+        type: plain
+        jaasConfigPassThrough:
+          secretRef: broker-credential
+      tls:
+        enabled: true
+  authorization:
+    type: rbac
+    superUsers:
+      - User:kafka
+  services:
+    mds:
+      tls:
+        enabled: true
+      tokenKeyPair:
+        secretRef: broker-credential
+      provider:
+        type: ldap
+        ldap:
+          address: ldap://ldap.sandbox.svc.cluster.local:389
+          authentication:
+            type: simple
+            simple:
+              secretRef: broker-credential
+          tls:
+            enabled: true
+          configurations:
+            groupNameAttribute: cn
+            groupObjectClass: groupOfNames
+            groupMemberAttribute: member
+            groupMemberAttributePattern: cn=(.*),ou=users,dc=test,dc=com
+            groupSearchBase: ou=groups,dc=test,dc=com
+            userNameAttribute: cn
+            userMemberOfAttributePattern: cn=(.*),ou=users,dc=test,dc=com
+            userObjectClass: organizationalRole
+            userSearchBase: ou=users,dc=test,dc=com
+  dependencies:
+    kafkaRest:
+      authentication:
+        type: bearer
+        bearer:
+          secretRef: broker-credential
+    zookeeper:
+      endpoint: zookeeper.sandbox.svc.cluster.local:2182
+      authentication:
+        type: digest
+        jaasConfig:
+          secretRef: broker-credential
+      tls:
+        enabled: true

examples/elk-logging/confluent/ksqldb.yaml

@@ -0,0 +1,29 @@
+---
+apiVersion: platform.confluent.io/v1beta1
+kind: KsqlDB
+metadata:
+  name: ksqldb
+spec:
+  authorization:
+    type: rbac
+  tls:
+    secretRef: tls-group1
+  dependencies:
+    kafka:
+      bootstrapEndpoint: kafka.sandbox.svc.cluster.local:9071
+      authentication:
+        type: plain
+        jaasConfig:
+          secretRef: mds-client-connect
+      tls:
+        enabled: true
+    mds:
+      endpoint: https://kafka.sandbox.svc.cluster.local:8090
+      tokenKeyPair:
+        secretRef: mds-public
+      authentication:
+        type: bearer
+        bearer:
+          secretRef: mds-client-ksqldb
+      tls:
+        enabled: true

examples/elk-logging/confluent/kustomization.yaml

@@ -0,0 +1,17 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: sandbox
+resources:
+- ldap.yaml
+- namespace.yaml
+- ../../../kustomize/base/confluent
+- ../../../kustomize/base/secrets-tls
+- ../../../kustomize/base/secrets-user
+patchesStrategicMerge:
+  - zookeeper.yaml
+  - kafka.yaml
+  - rest-class.yaml
+  - control-centre.yaml
+  - kafka-connect.yaml
+  - ksqldb.yaml
+  - schema-registry.yaml