Merge pull request #12 from osodevops/monitoring

Monitoring

Author: sionsmith

examples/monitoring/README.md

@@ -0,0 +1,46 @@
+# Monitoring (JMX/Prometheus/Grafana)
+In this example, we deploy an RBAC enabled Confluent cluster with Prometheus/Grafana integration.
+
+## Deploy Stack
+From within this present directory(./examples/monitoring), run the following command:
+
+```shell
+kubectl apply -k ../../kustomize/crds && sleep 1 && kubectl apply -k .
+```
+    
+Once all the the pods are in a 'Running' status, we can start to investigate the rest of the stack.
+
+## Prometheus
+Prometheus has a UI you can view by forwarding port 9090 with the following command, and then accessing `http://127.0.0.1:9090` from a local browser.
+```shell
+kubectl port-forward \
+$(kubectl get pods -n sandbox -l app=prometheus -l component=server -o name) \
+9090 --namespace sandbox
+```
+
+If you navigate to `http://localhost:9090/targets` you should hopefully see a screen such as this which indicates that Prometheus is successfully scraping from the Confluent Services
+![](../../resources/images/prometheus_targets.png)0
+
+The configuration file for Prometheus can be found at `./examples/monitoring/prometheus/server/cm.yaml` 
+
+## Grafana
+Grafana's Web UI runs on port 3000.  Similar to Prometheus, forward that port to your local machines with the following command:
+```shell
+kubectl port-forward \
+$(kubectl get pods -n sandbox kubectl get pods -n sandbox -l app.kubernetes.io/component=grafana -o name) \
+3000 --namespace sandbox
+```
+
+Navigate to http://localhost:3000
+
+You will need to login with the username 'admin' and the password 'password'.  At the home screen you will see a dashboard called 'Confluent Platform'.  Click on this
+![](../../resources/images/grafana-dashboard.png)
+
+This dashboard will display various metrics/alerts for the various Confluent services.  This dashboard has been captured in code, and can be edited at `./examples/grafana/dashboards.yaml`
+![](../../resources/images/grafana-dashboard2.png)
+
+
+
+
+
+

examples/monitoring/confluent/control-centre.yaml

@@ -0,0 +1,42 @@
+apiVersion: platform.confluent.io/v1beta1
+kind: ControlCenter
+metadata:
+  name: controlcenter
+spec:
+  authorization:
+    type: rbac
+  tls:
+    secretRef: tls-group1
+  dependencies:
+    kafka:
+      bootstrapEndpoint: kafka.sandbox.svc.cluster.local:9071
+      authentication:
+        type: plain
+        jaasConfig:
+          secretRef: mds-client-connect
+      tls:
+        enabled: true
+    mds:
+      endpoint: https://kafka.sandbox.svc.cluster.local:8090
+      tokenKeyPair:
+        secretRef: mds-public
+      authentication:
+        type: bearer
+        bearer:
+          secretRef: mds-client-c3
+      tls:
+        enabled: true
+    connect:
+      - name: connect
+        url:  https://connect.sandbox.svc.cluster.local:8083
+        tls:
+          enabled: true
+    ksqldb:
+      - name: ksqldb
+        url:  https://ksqldb.sandbox.svc.cluster.local:8088
+        tls:
+          enabled: true
+    schemaRegistry:
+      url: https://schemaregistry.sandbox.svc.cluster.local:8081
+      tls:
+        enabled: true

examples/monitoring/confluent/kafka-connect.yaml

@@ -0,0 +1,27 @@
+---
+apiVersion: platform.confluent.io/v1beta1
+kind: Connect
+metadata:
+  name: connect
+spec:
+  tls:
+    secretRef: tls-group1
+  authorization:
+    type: rbac
+  dependencies:
+    kafka:
+      bootstrapEndpoint: kafka.sandbox.svc.cluster.local:9071
+      authentication:
+        type: plain
+        jaasConfig:
+          secretRef: mds-client-connect
+      tls:
+        enabled: true
+    mds:
+      endpoint: https://kafka.sandbox.svc.cluster.local:8090
+      tokenKeyPair:
+        secretRef: mds-public
+      authentication:
+        type: bearer
+        bearer:
+          secretRef: mds-client-connect

examples/monitoring/confluent/kafka.yaml

@@ -0,0 +1,81 @@
+---
+apiVersion: platform.confluent.io/v1beta1
+kind: Kafka
+metadata:
+  name: kafka
+spec:
+  configOverrides:
+    server:
+      - confluent.schema.registry.url=https://schemaregistry.sandbox.svc.cluster.local:8081
+      - listener.name.internal.plain.sasl.server.callback.handler.class=io.confluent.security.auth.provider.ldap.LdapAuthenticateCallbackHandler
+      - listener.name.external.plain.sasl.server.callback.handler.class=io.confluent.security.auth.provider.ldap.LdapAuthenticateCallbackHandler
+      - listener.name.replication.plain.sasl.server.callback.handler.class=io.confluent.security.auth.provider.ldap.LdapAuthenticateCallbackHandler
+  tls:
+    secretRef: tls-group1
+  metricReporter:
+    enabled: true
+    authentication:
+      type: plain
+      jaasConfigPassThrough:
+        secretRef: broker-credential
+    tls:
+      enabled: true
+  listeners:
+    internal:
+      authentication:
+        type: plain
+        jaasConfigPassThrough:
+          secretRef: broker-credential
+      tls:
+        enabled: true
+    external:
+      authentication:
+        type: plain
+        jaasConfigPassThrough:
+          secretRef: broker-credential
+      tls:
+        enabled: true
+  authorization:
+    type: rbac
+    superUsers:
+      - User:kafka
+  services:
+    mds:
+      tls:
+        enabled: true
+      tokenKeyPair:
+        secretRef: broker-credential
+      provider:
+        type: ldap
+        ldap:
+          address: ldap://ldap.sandbox.svc.cluster.local:389
+          authentication:
+            type: simple
+            simple:
+              secretRef: broker-credential
+          tls:
+            enabled: true
+          configurations:
+            groupNameAttribute: cn
+            groupObjectClass: groupOfNames
+            groupMemberAttribute: member
+            groupMemberAttributePattern: cn=(.*),ou=users,dc=test,dc=com
+            groupSearchBase: ou=groups,dc=test,dc=com
+            userNameAttribute: cn
+            userMemberOfAttributePattern: cn=(.*),ou=users,dc=test,dc=com
+            userObjectClass: organizationalRole
+            userSearchBase: ou=users,dc=test,dc=com
+  dependencies:
+    kafkaRest:
+      authentication:
+        type: bearer
+        bearer:
+          secretRef: broker-credential
+    zookeeper:
+      endpoint: zookeeper.sandbox.svc.cluster.local:2182
+      authentication:
+        type: digest
+        jaasConfig:
+          secretRef: broker-credential
+      tls:
+        enabled: true

examples/monitoring/confluent/ksqldb.yaml

@@ -0,0 +1,29 @@
+---
+apiVersion: platform.confluent.io/v1beta1
+kind: KsqlDB
+metadata:
+  name: ksqldb
+spec:
+  authorization:
+    type: rbac
+  tls:
+    secretRef: tls-group1
+  dependencies:
+    kafka:
+      bootstrapEndpoint: kafka.sandbox.svc.cluster.local:9071
+      authentication:
+        type: plain
+        jaasConfig:
+          secretRef: mds-client-connect
+      tls:
+        enabled: true
+    mds:
+      endpoint: https://kafka.sandbox.svc.cluster.local:8090
+      tokenKeyPair:
+        secretRef: mds-public
+      authentication:
+        type: bearer
+        bearer:
+          secretRef: mds-client-ksqldb
+      tls:
+        enabled: true

examples/monitoring/confluent/kustomization.yaml

@@ -0,0 +1,17 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: sandbox
+resources:
+- ldap.yaml
+- namespace.yaml
+- ../../../kustomize/base/confluent
+- ../../../kustomize/base/secrets-tls
+- ../../../kustomize/base/secrets-user
+patchesStrategicMerge:
+  - zookeeper.yaml
+  - kafka.yaml
+  - rest-class.yaml
+  - control-centre.yaml
+  - kafka-connect.yaml
+  - ksqldb.yaml
+  - schema-registry.yaml