Data Lake configuration

examples/graylog-secret.yaml

@@ -0,0 +1,20 @@
+# Example Graylog Secret
+# If you wish to generate your own graylog-secret the following keys are required. All values should
+# be base64 encoded before applying to your cluster. Then supply the secret name as 
+# global.existingSecretName: ""
+#
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: graylog-secret-example
+data:
+  # Mongo DB URI with valid credentials and location of the Mongo cluster
+  GRAYLOG_MONGODB_URI: mongodb://admin:password@somewhere.local:27017/graylog_2?authSource=admin
+  # Graylog admin user name
+  GRAYLOG_ROOT_USERNAME: "some-user"
+  # Graylog admin user's password
+  GRAYLOG_PASSWORD_SECRET: "my-password"
+  # Graylog admin user's password SHA2 value. To get this value, see step 4 of 
+  # https://go2docs.graylog.org/current/downloading_and_installing_graylog/docker_installation.htm
+  GRAYLOG_ROOT_PASSWORD_SHA2: 6fa2288c361becce3e30ba4c41be7d8ba01e3580566f7acc76a7f99994474c46

examples/values-example-aws.yaml

@@ -3,7 +3,6 @@
 #
 # ALB Ingress Controller: https://kubernetes-sigs.github.io/aws-load-balancer-controller/v1.1/
 # External DNS: https://kubernetes-sigs.github.io/external-dns/latest/docs/tutorials/aws/
-# GP3 Storage: 
 #
 
 graylog:
@@ -16,8 +15,6 @@ graylog:
       enabled: true
       storageClass: "gp3"
       size: 5Gi
-  podAnnotations:
-    karpenter.sh/do-not-disrupt: "true"
   inputs:
     - port: 12201
       targetPort: 12201
@@ -38,74 +35,35 @@ datanode:
     karpenter.sh/do-not-disrupt: "true"
 
 ingress:
-  enabled: true
-  className: "alb"
-  annotations:
-    alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-1:000000000000:certificate/00000000-0000-0000-0000-000000000000
-    alb.ingress.kubernetes.io/group.name: some-graylog
-    alb.ingress.kubernetes.io/healthcheck-interval-seconds: "5"
-    alb.ingress.kubernetes.io/healthcheck-path: /
-    alb.ingress.kubernetes.io/healthcheck-port: "9000"
-    alb.ingress.kubernetes.io/healthcheck-protocol: HTTP
-    alb.ingress.kubernetes.io/healthcheck-timeout-seconds: "2"
-    alb.ingress.kubernetes.io/healthy-threshold-count: "2"
-    alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS": 443}]'
-    alb.ingress.kubernetes.io/load-balancer-attributes: "deletion_protection.enabled=false"
-    alb.ingress.kubernetes.io/scheme: internet-facing
-    alb.ingress.kubernetes.io/security-groups: sg-00000000000000000,sg-00000000000000001,sg-00000000000000002,sg-000000000000000003
-    alb.ingress.kubernetes.io/ssl-policy: ELBSecurityPolicy-TLS13-1-2-2021-06
-    alb.ingress.kubernetes.io/ssl-redirect: "443"
-    alb.ingress.kubernetes.io/success-codes: "200"
-    alb.ingress.kubernetes.io/target-type: ip
-    alb.ingress.kubernetes.io/unhealthy-threshold-count: "2"
-    external-dns.alpha.kubernetes.io/hostname: some-graylog.example.com
-  hosts:
-    - host: some-graylog.example.com
-      paths:
-        - path: /
-          pathType: Prefix
-          backend:
-            service:
-              name: graylog-app
-              port:
-                name: app
-
-
-
-# GLC Values
-# -- NOT PUBLIC -- 
-
-
-ingress:
-  enabled: true
-  className: "alb"
-  annotations:
-    alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-1:446604966667:certificate/1f6c138c-9715-4d3a-920d-a3778610a845
-    alb.ingress.kubernetes.io/group.name: helm-graylog-2
-    alb.ingress.kubernetes.io/healthcheck-interval-seconds: "5"
-    alb.ingress.kubernetes.io/healthcheck-path: /
-    alb.ingress.kubernetes.io/healthcheck-port: "9000"
-    alb.ingress.kubernetes.io/healthcheck-protocol: HTTP
-    alb.ingress.kubernetes.io/healthcheck-timeout-seconds: "2"
-    alb.ingress.kubernetes.io/healthy-threshold-count: "2"
-    alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS": 443}]'
-    alb.ingress.kubernetes.io/load-balancer-attributes: "deletion_protection.enabled=false"
-    alb.ingress.kubernetes.io/scheme: internet-facing
-    alb.ingress.kubernetes.io/security-groups: sg-05b42e0c22b938a38,sg-0d54f3c5e52d08a73,sg-0600c2f5354ca87dc,sg-044bf3d08385a571a
-    alb.ingress.kubernetes.io/ssl-policy: ELBSecurityPolicy-TLS13-1-2-2021-06
-    alb.ingress.kubernetes.io/ssl-redirect: "443"
-    alb.ingress.kubernetes.io/success-codes: "200"
-    alb.ingress.kubernetes.io/target-type: ip
-    alb.ingress.kubernetes.io/unhealthy-threshold-count: "2"
-    external-dns.alpha.kubernetes.io/hostname: helm-graylog-2.dev.graylog.cloud
-  hosts:
-    - host: helm-2.dev.graylog.cloud
-      paths:
-        - path: /
-          pathType: Prefix
-          backend:
-            service:
-              name: graylog-app
-              port:
-                name: app
-# -- NOT PUBLIC -- 
+  web:
+    enabled: true
+    className: "alb"
+    annotations:
+      alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-1:000000000000:certificate/00000000-0000-0000-0000-000000000000
+      alb.ingress.kubernetes.io/group.name: some-graylog
+      alb.ingress.kubernetes.io/healthcheck-interval-seconds: "5"
+      alb.ingress.kubernetes.io/healthcheck-path: /
+      alb.ingress.kubernetes.io/healthcheck-port: "9000"
+      alb.ingress.kubernetes.io/healthcheck-protocol: HTTP
+      alb.ingress.kubernetes.io/healthcheck-timeout-seconds: "2"
+      alb.ingress.kubernetes.io/healthy-threshold-count: "2"
+      alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS": 443}]'
+      alb.ingress.kubernetes.io/load-balancer-attributes: "deletion_protection.enabled=false"
+      alb.ingress.kubernetes.io/scheme: internet-facing
+      alb.ingress.kubernetes.io/security-groups: sg-00000000000000000,sg-00000000000000001,sg-00000000000000002,sg-000000000000000003
+      alb.ingress.kubernetes.io/ssl-policy: ELBSecurityPolicy-TLS13-1-2-2021-06
+      alb.ingress.kubernetes.io/ssl-redirect: "443"
+      alb.ingress.kubernetes.io/success-codes: "200"
+      alb.ingress.kubernetes.io/target-type: ip
+      alb.ingress.kubernetes.io/unhealthy-threshold-count: "2"
+      external-dns.alpha.kubernetes.io/hostname: some-graylog.example.com
+    hosts:
+      - host: some-graylog.example.com
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: graylog-app
+                port:
+                  name: app

graylog/templates/config/datanode.yaml

@@ -8,3 +8,7 @@ data:
   GRAYLOG_JAVA_OPTS: {{ .Values.datanode.config.javaOpts | quote }}
   GRAYLOG_SKIP_PREFLIGHT_CHECKS: {{ .Values.datanode.config.skipPreflightChecks | quote }}
   GRAYLOG_NODE_SEARCH_CACHE_SIZE: {{ .Values.datanode.config.nodeSearchCacheSize | quote }}
+  GRAYLOG_S3_CLIENT_DEFAULT_ENDPOINT: {{ .Values.datanode.config.s3ClientDefaultEndpoint | quote }}
+  GRAYLOG_S3_CLIENT_DEFAULT_REGION: {{ .Values.datanode.config.s3ClientDefaultRegion | quote }}
+  GRAYLOG_S3_CLIENT_DEFAULT_PROTOCOL: {{ .Values.datanode.config.s3ClientDefaultProtocol | quote }}
+  GRAYLOG_S3_CLIENT_DEFAULT_PATH_STYLE_ACCESS: {{ .Values.datanode.config.s3ClientDefaultPathStyleAccess | quote }}

graylog/templates/config/secret/secrets.yaml

@@ -88,6 +88,8 @@ data:
   GRAYLOG_ROOT_USERNAME: {{ .Values.graylog.config.rootUsername | default "admin" | b64enc }}
   GRAYLOG_PASSWORD_SECRET: {{ $graylogPepper }}
   GRAYLOG_ROOT_PASSWORD_SHA2: {{ $graylogSha }}
+  GRAYLOG_S3_CLIENT_DEFAULT_SECRET_KEY: {{ .Values.datanode.config.s3ClientDefaultSecretKey | quote }}
+  GRAYLOG_S3_CLIENT_DEFAULT_ACCESS_KEY: {{ .Values.datanode.config.s3ClientDefaultAccessKey | quote }}
 ---
 apiVersion: batch/v1
 kind: Job

graylog/templates/workload/statefulsets/datanode.yaml

@@ -77,6 +77,16 @@ spec:
                 secretKeyRef:
                   name: {{ include "graylog.secretsName" . }}
                   key: GRAYLOG_MONGODB_URI
+            - name: GRAYLOG_S3_CLIENT_DEFAULT_SECRET_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "graylog.secretsName" . }}
+                  key: GRAYLOG_S3_CLIENT_DEFAULT_SECRET_KEY
+            - name: GRAYLOG_S3_CLIENT_DEFAULT_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "graylog.secretsName" . }}
+                  key: GRAYLOG_S3_CLIENT_DEFAULT_ACCESS_KEY
           ports:
             - name: api
               containerPort: {{ .Values.datanode.custom.service.ports.api | default 8999 | int }}

graylog/values.yaml

@@ -153,6 +153,13 @@ datanode:
     javaOpts: "-Xms1g -Xmx1g"
     skipPreflightChecks: "false"
     nodeSearchCacheSize: "10gb"
+    s3ClientDefaultSecretKey: ""
+    s3ClientDefaultAccessKey: ""
+    s3ClientDefaultEndpoint: ""
+    s3ClientDefaultRegion: "us-east-2"
+    s3ClientDefaultProtocol: "http"
+    s3ClientDefaultPathStyleAccess: "true"
+
   # Custom Kubernetes-specific parameters
   custom:
     podAnnotations: {}