Feature/enable tls (#20)

* feat(tls): remove readonlyrest plugin and config

* feat(tls): add certs generation docker compose

* also rename network so it's not efk_efk

* feat(tls): fix health checks

* chore: bump version for sarif output

---------

Co-authored-by: Lee Myring <[email protected]>

Author: deploymentking

.github/workflows/rubocop-analysis.yml

@@ -30,6 +30,6 @@ jobs:
         "
 
     - name: Upload Sarif output
-      uses: github/codeql-action/upload-sarif@v1
+      uses: github/codeql-action/upload-sarif@v2
       with:
         sarif_file: rubocop.sarif

.gitignore

@@ -24,5 +24,4 @@ target
 coverage
 VERSION
 output.log
-
 es-passwords.txt

Rakefile

@@ -9,6 +9,11 @@ task :all_sources do
   sh 'docker ps -a --format "table {{.ID}}\t{{.Status}}\t{{.Names}}\t{{.Ports}}"'
 end
 
+desc 'Generate the certs Docker volume'
+task :certs do
+  sh 'docker-compose -f docker-compose-create-certs.yml run --rm create_certs'
+end
+
 desc 'Clean some generated files'
 task :clean do
   %w[
@@ -23,22 +28,32 @@ task :clean do
   ].each { |f| FileUtils.rm_rf(Dir.glob(f)) }
 end
 
-desc 'Stop the entire EFK stack, any additional sources and the minikube cluster'
+
+desc 'Start the client instances Kibana and ElasticHQ (Elasticsearch cluster must be up and running first)'
+task :clients do
+  trap('SIGINT') do
+    puts 'Cancelled Kibana launch...'
+    exit
+  end
+  sh './scripts/start-clients.sh'
+end
+
+desc 'Kill the entire EFK stack, any additional sources and the minikube cluster'
 task :down do
-  sh './scripts/stop-efk.sh || true'
+  sh './scripts/down-es-cluster.sh || true'
 end
 
-desc 'Start the EFK stack components (including elasticHQ)'
-task :efk do
+desc 'Start the Elasticsearch cluster (including elasticHQ)'
+task :elasticsearch do
   trap('SIGINT') do
-    puts 'Cancelled EFK stack launch...'
+    puts 'Cancelled Elasticsearch cluster launch...'
     exit
   end
-  sh './scripts/start-efk.sh'
+  sh './scripts/start-es-cluster.sh'
 end
 
 desc 'Run ALL the rake tasks: clean test and build'
-task everything: %w[down clean style test efk k8s all_sources]
+task everything: %w[down clean style test elasticsearch kibana k8s all_sources]
 
 desc 'Start the Kubernetes Minikube components'
 task :k8s do
@@ -59,6 +74,15 @@ task :logs do
   sh 'docker-compose logs -f'
 end
 
+task :passwords do
+  sh '
+    docker exec elasticsearch-master /bin/bash \
+      -c "bin/elasticsearch-setup-passwords auto \
+      --batch \
+      --url https://elasticsearch:9200" > es-passwords.txt
+  '
+end
+
 desc 'Start the Prometheus stack component'
 task :prometheus do
   trap('SIGINT') do
@@ -93,6 +117,11 @@ task :start, :source do |_task, args|
   sh "./scripts/start-source.sh #{args[:source]}"
 end
 
+desc 'Stop the EFK cluster'
+task :stop do
+  sh './scripts/stop-es-cluster.sh'
+end
+
 desc 'Run all style checks'
 task style: %w[rubocop]
 

docker-compose-clients.yml

@@ -0,0 +1,47 @@
+version: '3.6'
+
+services:
+  kibana:
+    build:
+      context: ./kibana
+      args:
+        VERSION_EFK: ${VERSION_EFK}
+    container_name: kibana
+    environment:
+      ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_KIBANA_PASSWORD}
+      SERVER_SSL_KEYPASSPHRASE: ${KEY_PASSPHRASE}
+    hostname: kibana
+    networks:
+      - cluster
+    ports:
+      - 5601:5601
+    volumes:
+      - certs:/usr/share/kibana/config/certificates
+      - ./kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml
+
+  elastichq:
+    build:
+      context: ./elasticHQ
+      args:
+        VERSION_ELASTICHQ: ${VERSION_ELASTICHQ}
+    container_name: elastichq
+    environment:
+      HQ_DEFAULT_URL: https://elastic:${ELASTICSEARCH_ELASTIC_PASSWORD}@elasticsearch:9200
+      HQ_ENABLE_SSL: "True"
+      HQ_CA_CERTS: /usr/share/elastichq/ca/ca.crt
+      HQ_VERIFY_CERTS: "False"
+    hostname: elastichq
+    networks:
+      - cluster
+    ports:
+      - 5000:5000
+    volumes:
+      - certs:/usr/share/elastichq
+
+networks:
+  cluster:
+    driver: bridge
+
+volumes:
+  certs:
+    driver: local

docker-compose-create-certs.yml

@@ -30,12 +30,12 @@ services:
       - certs:/certs
       - ./elasticsearch/certs/instances.yml:/usr/share/elasticsearch/instances.yml
     networks:
-      - efk
+      - cluster
 
 volumes:
   certs:
     driver: local
 
 networks:
-  efk:
+  cluster:
     driver: bridge

docker-compose.yml

@@ -11,14 +11,10 @@ services:
     environment:
       - ${ES_JAVA_OPTS:-/dev/null}
       - KEY_PASSPHRASE=${KEY_PASSPHRASE}
-    healthcheck:
-      test: curl --cacert $CERTS_DIR/ca.crt -s https://elasticsearch:9200 >/dev/null; if [[ $$? == 52 ]]; then echo 0; else echo 1; fi
-      interval: 30s
-      timeout: 10s
-      retries: 5
+      - CERTS_DIR=${CERTS_DIR}
     hostname: elasticsearch
     networks:
-      - efk
+      - cluster
     ports:
       - "9200:9200"
       - "9300:9300"
@@ -33,8 +29,7 @@ services:
       - certs:$CERTS_DIR
       - elasticsearch_master:/usr/share/elasticsearch/data
       - ./elasticsearch/entrypoint.sh:/usr/share/elasticsearch/custom-entrypoint.sh
-      - ./elasticsearch/config/master/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
-      - ./elasticsearch/config/readonlyrest.yml:/usr/share/elasticsearch/config/readonlyrest.yml
+      - ./elasticsearch/config/master.yml:/usr/share/elasticsearch/config/elasticsearch.yml
 
   elasticsearch-hot:
     build:
@@ -48,7 +43,7 @@ services:
       - KEY_PASSPHRASE=${KEY_PASSPHRASE}
     hostname: elasticsearch-hot
     networks:
-      - efk
+      - cluster
     ulimits:
       nofile:
         soft: 65536
@@ -60,8 +55,7 @@ services:
       - certs:$CERTS_DIR
       - elasticsearch_hot:/usr/share/elasticsearch/data
       - ./elasticsearch/entrypoint.sh:/usr/share/elasticsearch/custom-entrypoint.sh
-      - ./elasticsearch/config/hot/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
-      - ./elasticsearch/config/readonlyrest.yml:/usr/share/elasticsearch/config/readonlyrest.yml
+      - ./elasticsearch/config/hot.yml:/usr/share/elasticsearch/config/elasticsearch.yml
 
   elasticsearch-warm:
     build:
@@ -75,7 +69,7 @@ services:
       - KEY_PASSPHRASE=${KEY_PASSPHRASE}
     hostname: elasticsearch-warm
     networks:
-      - efk
+      - cluster
     ulimits:
       nofile:
         soft: 65536
@@ -87,8 +81,7 @@ services:
       - certs:$CERTS_DIR
       - elasticsearch_warm:/usr/share/elasticsearch/data
       - ./elasticsearch/entrypoint.sh:/usr/share/elasticsearch/custom-entrypoint.sh
-      - ./elasticsearch/config/warm/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
-      - ./elasticsearch/config/readonlyrest.yml:/usr/share/elasticsearch/config/readonlyrest.yml
+      - ./elasticsearch/config/warm.yml:/usr/share/elasticsearch/config/elasticsearch.yml
 
 #  fluentd:
 #    build:
@@ -103,7 +96,7 @@ services:
 #      - ./fluentd/fluentd.properties
 #    hostname: fluentd
 #    networks:
-#      - efk
+#      - cluster
 #    ports:
 #      - "5140:5140"
 #      - "5140:5140/udp"
@@ -116,50 +109,9 @@ services:
 #    volumes:
 #      - ./fluentd/config:/fluentd/etc
 #      - ./fluentd/certs:/fluentd/certs
-#
-  kibana:
-    build:
-      context: ./kibana
-      args:
-        VERSION_EFK: ${VERSION_EFK}
-    container_name: kibana
-    depends_on:
-      - elasticsearch
-    environment:
-      ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD}
-      SERVER_SSL_KEYPASSPHRASE: ${KEY_PASSPHRASE}
-    hostname: kibana
-    networks:
-      - efk
-    ports:
-      - 5601:5601
-    volumes:
-      - certs:/usr/share/kibana/config/certificates
-      - ./kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml
-
-#  elastichq:
-#    build:
-#      context: ./elasticHQ
-#      args:
-#        VERSION_ELASTICHQ: ${VERSION_ELASTICHQ}
-#    container_name: elastichq
-#    depends_on:
-#      - elasticsearch
-#    environment:
-#      HQ_DEFAULT_URL: https://elasticsearch:9200
-#      HQ_ENABLE_SSL: "True"
-#      HQ_CA_CERTS: /usr/share/elastichq/ca/ca.crt
-#      HQ_VERIFY_CERTS: "False"
-#    hostname: elastichq
-#    networks:
-#      - efk
-#    ports:
-#      - 5000:5000
-#    volumes:
-#      - certs:/usr/share/elastichq
 
 networks:
-  efk:
+  cluster:
     driver: bridge
 
 volumes:

elastichq/Dockerfile

@@ -6,6 +6,4 @@ LABEL Description="ElasticHQ instance"
 LABEL Version="2.0.0"
 
 EXPOSE 5000
-
-HEALTHCHECK --interval=5s --timeout=2s --retries=15 \
-    CMD nc -z localhost 5000 || exit 1
+HEALTHCHECK --interval=5s --timeout=2s --retries=15 CMD nc -z localhost 5000 || exit 1

elasticsearch/Dockerfile

@@ -6,6 +6,4 @@ LABEL Description="Elasticsearch instance"
 LABEL Version="2.0.0"
 
 EXPOSE 9200 9300
-
-HEALTHCHECK --interval=5s --timeout=2s --retries=15 \
-    CMD curl --silent --fail localhost:9200/_cluster/health || exit 1
+HEALTHCHECK --interval=5s --timeout=2s --retries=15 CMD nc -z elasticsearch 9200 || exit 1