[filebeat] using https to connect to elasticsearch (#1627)

* Adding support to Ingress networking.k8s.io/v1

* Adjusting ES service name

* Removing ingress typo & adjusting python test

* Adjusting python tests to use the new ingress version

* fixing conflict

* Adding support to kubernetes ingress v1 & ClassName

* Adding reformatted files

* fixing conflict

* Adding ClassName & Pathtype on ingress settings

* Performing syntax adjustments and removing comments

* adjusting settings to connect into elasticsearch using https

* adjusting tests

* adjusting tests

* adjusting tests

* adding body, username, password and increasing version upgraded

* fix goss syntax

Co-authored-by: jmlrt <8582351+jmlrt@users.noreply.github.com>

Author: framsouza

filebeat/examples/default/test/goss.yaml

@@ -25,9 +25,10 @@ user:
     gid: 1000
 
 http:
-  http://elasticsearch-master:9200/_cat/indices:
+  https://elasticsearch-master:9200/_cat/indices:
     status: 200
     timeout: 2000
+    allow-insecure: true
     username: "{{ .Env.ELASTICSEARCH_USERNAME }}"
     password: "{{ .Env.ELASTICSEARCH_PASSWORD }}"
     body:
@@ -45,4 +46,4 @@ command:
   cd /usr/share/filebeat && filebeat test output:
     exit-status: 0
     stdout:
-      - "elasticsearch: http://elasticsearch-master:9200"
+      - "elasticsearch: https://elasticsearch-master:9200"

filebeat/examples/deployment/test/goss.yaml

@@ -1,6 +1,7 @@
 http:
-  http://elasticsearch-master:9200/_cat/indices:
+  https://elasticsearch-master:9200/_cat/indices:
     status: 200
+    allow-insecure: true
     timeout: 2000
     username: "{{ .Env.ELASTICSEARCH_USERNAME }}"
     password: "{{ .Env.ELASTICSEARCH_PASSWORD }}"

filebeat/examples/oss/test/goss.yaml

@@ -15,7 +15,8 @@ user:
     gid: 1000
 
 http:
-  http://elasticsearch-master:9200/_cat/indices:
+  https://elasticsearch-master:9200/_cat/indices:
+    allow-insecure: true
     status: 200
     timeout: 2000
     username: "{{ .Env.ELASTICSEARCH_USERNAME }}"

filebeat/examples/oss/values.yaml

@@ -15,10 +15,16 @@ daemonset:
                 logs_path: "/var/log/containers/"
       output.elasticsearch:
         host: '${NODE_NAME}'
-        hosts: "elasticsearch-master:9200"
+        hosts: ["https://elasticsearch-master:9200"]
         username: '${ELASTICSEARCH_USERNAME}'
         password: '${ELASTICSEARCH_PASSWORD}'
         index: "filebeat-oss-%{[agent.version]}-%{+yyyy.MM.dd}"
+        ssl.certificate_authorities:
+          - /usr/share/filebeat/certs/ca.crt
       setup.ilm.enabled: false
       setup.template.name: "filebeat"
       setup.template.pattern: "filebeat-oss-*"
+  secretMounts:
+    - name: elasticsearch-master-certs
+      secretName: elasticsearch-master-certs
+      path: /usr/share/filebeat/certs

filebeat/examples/upgrade/Makefile

@@ -4,8 +4,8 @@ include ../../../helpers/examples.mk
 
 CHART := filebeat
 RELEASE := helm-filebeat-upgrade
-# K8S 1.22 doesn't support anymore rbac.authorization.k8s.io/v1beta1 used in 7.9.0
-FROM := 7.10.0
+# upgrade from versions before 7.17.1 isn't compatible with 8.x
+FROM := 7.17.1
 
 install:
 	../../../helpers/upgrade.sh --chart $(CHART) --release $(RELEASE) --from $(FROM)

filebeat/examples/upgrade/test/goss.yaml

@@ -25,11 +25,14 @@ user:
     gid: 1000
 
 http:
-  http://upgrade-master:9200/_cat/indices:
+  https://upgrade-master:9200/_cat/indices:
     status: 200
+    allow-insecure: true
     timeout: 2000
     body:
       - "filebeat-8.1.0"
+    username: "{{ .Env.ELASTICSEARCH_USERNAME }}"
+    password: "{{ .Env.ELASTICSEARCH_PASSWORD }}"
 
 file:
   /usr/share/filebeat/filebeat.yml:
@@ -42,4 +45,4 @@ command:
   cd /usr/share/filebeat && filebeat test output:
     exit-status: 0
     stdout:
-      - "elasticsearch: http://upgrade-master:9200"
+      - "elasticsearch: https://upgrade-master:9200"

filebeat/examples/upgrade/values.yaml

@@ -1,15 +1,39 @@
----
-daemonset:
-  extraEnvs:
-    - name: ELASTICSEARCH_HOSTS
-      value: upgrade-master:9200
-    - name: "ELASTICSEARCH_USERNAME"
-      valueFrom:
-        secretKeyRef:
-          name: upgrade-master-credentials
-          key: username
-    - name: "ELASTICSEARCH_PASSWORD"
-      valueFrom:
-        secretKeyRef:
-          name: upgrade-master-credentials
-          key: password
+extraEnvs:
+  - name: "ELASTICSEARCH_HOSTS"
+    value: "https://upgrade-master:9200"
+  - name: "ELASTICSEARCH_USERNAME"
+    valueFrom:
+      secretKeyRef:
+        name: upgrade-master-credentials
+        key: username
+  - name: "ELASTICSEARCH_PASSWORD"
+    valueFrom:
+      secretKeyRef:
+        name: upgrade-master-credentials
+        key: password
+  - name: ssl.certificate_authorities
+    value: "/usr/share/filebeat/certs/ca.crt"
+filebeatConfig:
+  filebeat.yml: |
+    filebeat.inputs:
+    - type: container
+      paths:
+        - /var/log/containers/*.log
+      processors:
+      - add_kubernetes_metadata:
+          host: ${NODE_NAME}
+          matchers:
+          - logs_path:
+              logs_path: "/var/log/containers/"
+    output.elasticsearch:
+      host: '${NODE_NAME}'
+      hosts: '${ELASTICSEARCH_HOSTS:upgrade-master:9200}'
+      username: '${ELASTICSEARCH_USERNAME}'
+      password: '${ELASTICSEARCH_PASSWORD}'
+      protocol: https
+      ssl.certificate_authorities:
+      - /usr/share/filebeat/certs/ca.crt
+secretMounts:
+  - name: upgrade-master-certs
+    secretName: upgrade-master-certs
+    path: /usr/share/filebeat/certs

filebeat/values.yaml

@@ -48,15 +48,20 @@ daemonset:
 
       output.elasticsearch:
         host: '${NODE_NAME}'
-        hosts: '${ELASTICSEARCH_HOSTS:elasticsearch-master:9200}'
+        hosts: '["https://${ELASTICSEARCH_HOSTS:elasticsearch-master:9200}"]'
         username: '${ELASTICSEARCH_USERNAME}'
         password: '${ELASTICSEARCH_PASSWORD}'
+        protocol: https
+        ssl.certificate_authorities: ["/usr/share/filebeat/certs/ca.crt"]
   # Only used when updateStrategy is set to "RollingUpdate"
   maxUnavailable: 1
   nodeSelector: {}
   # A list of secrets and their paths to mount inside the pod
   # This is useful for mounting certificates for security other sensitive values
-  secretMounts: []
+  secretMounts: 
+   - name: elasticsearch-master-certs
+     secretName: elasticsearch-master-certs
+     path: /usr/share/filebeat/certs/
   #  - name: filebeat-certificates
   #    secretName: filebeat-certificates
   #    path: /usr/share/filebeat/certs
@@ -117,13 +122,18 @@ deployment:
 
       output.elasticsearch:
         host: "${NODE_NAME}"
-        hosts: "${ELASTICSEARCH_HOSTS:elasticsearch-master:9200}"
+        hosts: '["https://${ELASTICSEARCH_HOSTS:elasticsearch-master:9200}"]'
         username: "${ELASTICSEARCH_USERNAME}"
         password: "${ELASTICSEARCH_PASSWORD}"
+        protocol: https
+        ssl.certificate_authorities: ["/usr/share/filebeat/certs/ca.crt"]
   nodeSelector: {}
   # A list of secrets and their paths to mount inside the pod
   # This is useful for mounting certificates for security other sensitive values
-  secretMounts: []
+  secretMounts:
+   - name: elasticsearch-master-certs
+     secretName: elasticsearch-master-certs
+     path: /usr/share/filebeat/certs/
   #  - name: filebeat-certificates
   #    secretName: filebeat-certificates
   #    path: /usr/share/filebeat/certs