Merge pull request #128 from cloudfoundry-community/es6

Infra-Red · web-flow · commit 4cdd1ac272f8 · 2018-11-30T16:02:36.000+03:00
Upgrade to ES6
diff --git a/README.md b/README.md
@@ -11,11 +11,17 @@ Logsearch < v23.0.0 was based on Elasticsearch 1.x and Kibana 3.
 
 Logsearch > v200 is based on Elasticsearch 2.x and Kibana 4.
 
-There is NO upgrade path from Elasticsearch 1.x to 2.x.  Sorry :(
+- There is NO upgrade path from Elasticsearch 1.x to 2.x.  Sorry :(
 
-Logsearch > v204.0.0 is based on Elasticsearch 5.x and Kibana 5.
+Logsearch > v204.0.0 is based on Elastic stack version 5.
+
+- For upgrade procedure from Elasticsearch 2.x please refer to [v205.0.0 release notes](https://github.com/cloudfoundry-community/logsearch-boshrelease/releases/tag/v205.0.0#component-updates).
+
+Logsearch > v210.0.0 is based on Elastic stack version 6.
+
+- Elasticsearch 6.x can use indices created in Elasticsearch 5.x, but not those created in Elasticsearch 2.x or before.
+- **Important**: After upgrading running 5.x cluster to 6.x all existing indicies will be available for reading data. However, writing to these indicies is not possible. In order to write data immediatelly after upgrade you have to [change index naming convention](https://github.com/cloudfoundry-community/logsearch-boshrelease/commit/2f83b41ee14dbe3141e21cc0c40df340d50e0169). As long as index names are usually based on current date, this change can be safely reverted in a day or so.
 
-For upgrade procedure from Elasticsearch 2.x please refer to [v205.0.0 release notes](https://github.com/cloudfoundry-community/logsearch-boshrelease/releases/tag/v205.0.0#component-updates).
 
 ## Getting Started
 
diff --git a/config/blobs.yml b/config/blobs.yml
@@ -30,10 +30,10 @@ curator/vendor/voluptuous-0.10.5.tar.gz:
   size: 41436
   object_id: b67881a7-7adb-4456-7d0b-f5b00e936152
   sha: 31373cc4ba935eda76b55e64e8bd218ce4882e2b
-elasticsearch/elasticsearch-5.6.13.tar.gz:
-  size: 33894848
-  object_id: 19557b81-5d85-4d8b-7d80-1ecb540b145b
-  sha: 4c8ed27d362d76b5bdc52327721b44bb55215205
+elasticsearch/elasticsearch-6.4.3.tar.gz:
+  size: 97872736
+  object_id: 23a715d7-4f0c-48c4-41f4-8090f6148a11
+  sha: 57f2d6066b1277ab575e252044bee0ea9fd1b979
 haproxy/haproxy-1.7.5.tar.gz:
   size: 1743979
   object_id: 27004bb4-29f6-493f-80ec-6a4170a955d5
@@ -42,18 +42,18 @@ haproxy/pcre-8.40.tar.gz:
   size: 2065161
   object_id: edf51dad-4c6c-4a3d-8ebb-3a9b9f9641e5
   sha: 10384eb3d411794cc15f55b9d837d3f69e35391e
-java8/openjdk-1.8.0_152.tar.gz:
-  size: 45661256
-  object_id: 2211b342-4102-4587-6037-23a143726f2b
-  sha: 0d5758ed6f0e040684ea8bdf410ba44422f29f47
-kibana/kibana-5.6.13-linux-x86_64.tar.gz:
-  size: 53161803
-  object_id: f305ce78-cc27-4fc3-65e6-7351c78bed22
-  sha: 5562635acea8b262dc47e2c09f25572e30cef950
-logstash/logstash-5.6.13.tar.gz:
-  size: 104517136
-  object_id: 16463798-cf0d-46b9-43b9-abbe235737b4
-  sha: f83fb75409e63c5e0450f22d5d14a7c3c5af3192
+java8/openjdk-1.8.0_192.tar.gz:
+  size: 45305426
+  object_id: 0a033904-08ea-41d5-6191-62a634526d8f
+  sha: dae286f0e11c1a401e2c84a371f5cc2c6e2637ef
+kibana/kibana-6.4.3-linux-x86_64.tar.gz:
+  size: 235299366
+  object_id: 0aad6732-1974-4211-6661-6b064d324644
+  sha: 30d9adca00c0c4bae35b40a305fcb16a046a9601
+logstash/logstash-6.4.3.tar.gz:
+  size: 153936585
+  object_id: 8d5c9c76-018d-4dec-6a31-5dc2c8018c57
+  sha: cb1c4c6ff2c5ef9b14809733624b5c4b5e55e78d
 logstash/logstash-filter-alter-3.0.2.zip:
   size: 7425
   object_id: 543e5010-6449-400b-489d-31b82dcb4670
diff --git a/deployment/logsearch-deployment.yml b/deployment/logsearch-deployment.yml
@@ -33,6 +33,12 @@ instance_groups:
       elasticsearch:
         node:
           allow_master: true
+        config_options:
+          xpack.monitoring.enabled: false
+          xpack.graph.enabled: false
+          xpack.ml.enabled: false
+          xpack.security.enabled: false
+          xpack.watcher.enabled: false
   - name: syslog_forwarder
     release: logsearch
     consumes:
@@ -172,6 +178,12 @@ instance_groups:
       elasticsearch:
         node:
           allow_data: true
+        config_options:
+          xpack.monitoring.enabled: false
+          xpack.graph.enabled: false
+          xpack.ml.enabled: false
+          xpack.security.enabled: false
+          xpack.watcher.enabled: false
   - name: syslog_forwarder
     release: logsearch
     consumes:
@@ -210,10 +222,15 @@ instance_groups:
     properties:
       kibana:
         health:
-          timeout: 300
+          timeout: 500
         env:
           - NODE_ENV: production
-          
+        config_options:
+          xpack.monitoring.enabled: false
+          xpack.graph.enabled: false
+          xpack.ml.enabled: false
+          xpack.security.enabled: false
+          xpack.watcher.enabled: false
   - name: syslog_forwarder
     release: logsearch
     consumes:
@@ -292,7 +309,7 @@ releases:
 
 stemcells:
 - alias: default
-  os: ubuntu-trusty
+  os: ubuntu-xenial
   version: "latest"
 
 variables: []
diff --git a/deployment/operations/README.md b/deployment/operations/README.md
@@ -28,4 +28,10 @@ Includes [logsearch-for-cloudfoundry](https://github.com/cloudfoundry-community/
 
 To use this extention, you have to provide following variables: `cf_admin_password`, `uaa_admin_client_secret` and `system_domain`. You can find them in your CloudFoundry deployment variables.
 
-After deployment, please run `create-uaa-client` errand to create UAA client. After that, you will be able to login into Logsearch UI using your CloudFoundry credentials, and use `https://logs.<YOUR_DOMAIN>` endpoint to access Logsearch UI.
+After deployment, please run `create-uaa-client` errand to create UAA client. After that, you will be able to login into Logsearch UI using your CloudFoundry credentials, and use `https://logs.<YOUR_DOMAIN>` endpoint to access Logsearch UI.
+
+## cf-kibana.yml
+Errand to push Kibana application into CloudFoundry as an alternative to running it as a separate job.
+
+- Depend from **cloudfoundry.yml**
+- Requires `cf-kibana_client_secre` variable
diff --git a/deployment/operations/cf-kibana.yml b/deployment/operations/cf-kibana.yml
@@ -0,0 +1,28 @@
+---
+- type: replace
+  path: /instance_groups/name=maintenance/jobs/-
+  value:
+    name: cf-kibana
+    release: logsearch-for-cloudfoundry
+    lifecycle: errand
+    consumes:
+      elasticsearch: {from: elasticsearch_master}
+      cloud_controller: {from: cloud_controller, deployment: cf}
+    properties:
+      cloudfoundry:
+        admin_password: "((cf_admin_password))"
+      cf-kibana:
+        app_name: kibana
+        app_memory: 4G
+        oauth2_client_secret: "((cf-kibana_client_secret))"
+        cloudfoundry:
+          uaa_admin_client_secret: "((uaa_admin_client_secret))"
+          api_security_group: public_networks
+          system_org: system
+          apps_domain: ((system_domain))
+        config_options:
+          xpack.monitoring.enabled: false
+          xpack.graph.enabled: false
+          xpack.ml.enabled: false
+          xpack.security.enabled: false
+          xpack.watcher.enabled: false
diff --git a/deployment/operations/cloudfoundry.yml b/deployment/operations/cloudfoundry.yml
@@ -72,7 +72,7 @@
 - type: replace
   path: /instance_groups/name=ingestor/jobs/name=ingestor_syslog/properties/logstash_parser/elasticsearch?
   value:
-    index: logs-%{[@metadata][index]}-%{+YYYY.MM.dd}
+    index: logs-%{[@metadata][index]}-v6-%{+YYYY.MM.dd}
 
 - type: replace
   path: /instance_groups/name=ingestor/jobs/-
diff --git a/jobs/archiver_syslog/spec b/jobs/archiver_syslog/spec
@@ -13,6 +13,7 @@ templates:
   config/syslog_tls.crt.erb: config/syslog_tls.crt
   config/syslog_tls.key.erb: config/syslog_tls.key
   config/logstash.yml.erb: config/logstash.yml
+  config/jvm.options.erb: config/jvm.options
   data/properties.sh.erb: data/properties.sh
   helpers/ctl_setup.sh: helpers/ctl_setup.sh
   helpers/ctl_utils.sh: helpers/ctl_utils.sh
@@ -33,6 +34,9 @@ provides:
 properties:
   logstash.heap_size:
     description: sets jvm heap sized
+  logstash.jvm_options:
+    description: additional jvm options
+    default: []
   logstash.metadata_level:
     description: "Whether to include additional metadata throughout the event lifecycle. NONE = disabled, DEBUG = fully enabled"
     default: "NONE"
diff --git a/jobs/archiver_syslog/templates/bin/archiver_syslog_ctl b/jobs/archiver_syslog/templates/bin/archiver_syslog_ctl
@@ -41,6 +41,13 @@ case $1 in
     # construct a complete config file from all the fragments
     cat ${JOB_DIR}/config/input_and_output.conf > ${JOB_DIR}/config/logstash.conf
 
+    # clear persistent queue if the upgrade failed last run
+    if tail -8 $LOG_DIR/$JOB_NAME.stdout.log | grep 'QueueUpgrade - Logstash was unable to upgrade your persistent queue data' >/dev/null ; then
+      mkdir ${STORE_DIR}/oldqueue.$$
+      mv ${STORE_DIR}/queue ${STORE_DIR}/.lock ${STORE_DIR}/dead_letter_queue ${STORE_DIR}/uuid ${STORE_DIR}/oldqueue.$$
+    fi
+
+    ulimit -n <%= p("logstash_archiver.files") %>
     exec chpst -u vcap:vcap /var/vcap/packages/logstash/bin/logstash \
         --path.data ${STORE_DIR} \
         --path.config ${JOB_DIR}/config/logstash.conf \
diff --git a/jobs/archiver_syslog/templates/config/jvm.options.erb b/jobs/archiver_syslog/templates/config/jvm.options.erb
@@ -0,0 +1,4 @@
+# the default /tmp dir is often noexec, so we need to use one that is not.
+-Djava.io.tmpdir=/var/vcap/data/sys/tmp/archiver_syslog
+
+<% p('logstash.jvm_options').each do |opt| %><%= opt %><% end %>
diff --git a/jobs/elasticsearch/templates/bin/elasticsearch_ctl b/jobs/elasticsearch/templates/bin/elasticsearch_ctl
@@ -19,7 +19,7 @@ export HEAP_SIZE=$((( $( cat /proc/meminfo | grep MemTotal | awk '{ print $2 }'
 <% end %>
 
 export ES_JAVA_OPTS="-Xms$HEAP_SIZE -Xmx$HEAP_SIZE -Djava.io.tmpdir=$TMP_DIR"
-export ES_JVM_OPTIONS="${JOB_DIR}/config/jvm.options"
+export ES_PATH_CONF=${JOB_DIR}/config
 # encountered environment variables that are no longer suppor; thentted
 
 export MAX_OPEN_FILES=<%= p("elasticsearch.limits.fd") %>
@@ -66,7 +66,6 @@ case $1 in
 
     chpst -u vcap:vcap /var/vcap/packages/elasticsearch/bin/elasticsearch \
          -p ${PIDFILE} \
-         -Epath.conf=${JOB_DIR}/config \
          <%= p("elasticsearch.exec.options", []).join(' ') %> \
          >>$LOG_DIR/$JOB_NAME.stdout.log \
          2>>$LOG_DIR/$JOB_NAME.stderr.log
diff --git a/jobs/elasticsearch/templates/config/config.yml.erb b/jobs/elasticsearch/templates/config/config.yml.erb
@@ -1,9 +1,7 @@
 bootstrap.memory_lock: true
 
-path.conf: "/var/vcap/jobs/elasticsearch/config"
 path.logs: "/var/vcap/sys/log/elasticsearch"
 path.data: "/var/vcap/store/elasticsearch"
-path.scripts: "/var/vcap/data/elasticsearch/plugin-scripts"
 <% if p('elasticsearch.path_repo') != '' %>
 path.repo: ["<%= p('elasticsearch.path_repo') %>"]
 <% end %>
diff --git a/jobs/elasticsearch/templates/helpers/ctl_setup.sh b/jobs/elasticsearch/templates/helpers/ctl_setup.sh
@@ -56,6 +56,9 @@ do
 done
 export TMPDIR=$TMP_DIR
 
+# need this so that java keystore can be created (https://github.com/elastic/ansible-elasticsearch/issues/430)
+chmod 1775 ${JOB_DIR}/config
+
 export C_INCLUDE_PATH=/var/vcap/packages/mysqlclient/include/mysql:/var/vcap/packages/sqlite/include:/var/vcap/packages/libpq/include
 export LIBRARY_PATH=/var/vcap/packages/mysqlclient/lib/mysql:/var/vcap/packages/sqlite/lib:/var/vcap/packages/libpq/lib
 
diff --git a/jobs/elasticsearch_config/spec b/jobs/elasticsearch_config/spec
@@ -6,7 +6,7 @@ packages:
   - java8
 
 templates:
-  bin/run.erb: bin/run
+  bin/run.erb: bin/upload-config
   bin/job.process: bin/job.process
   index-templates/shards-and-replicas.json.erb: index-templates/shards-and-replicas.json
   index-templates/index-settings.json.erb: index-templates/index-settings.json
diff --git a/jobs/elasticsearch_config/templates/bin/job.process b/jobs/elasticsearch_config/templates/bin/job.process
@@ -20,7 +20,7 @@ case $1 in
   start)
     echo $$ > "${PIDFILE}"
 
-    /var/vcap/jobs/elasticsearch_config/bin/run
+    /var/vcap/jobs/elasticsearch_config/bin/upload-config
 
     # pretend we're still running to keep monit happy
     echo 1 > "${PIDFILE}"
diff --git a/jobs/elasticsearch_config/templates/index-templates/index-mappings.json.erb b/jobs/elasticsearch_config/templates/index-templates/index-mappings.json.erb
@@ -9,7 +9,7 @@
     }    
   },
   "mappings": {
-    "_default_": {
+    "doc": {
       "_all": {
         "enabled": false
       },
@@ -19,21 +19,21 @@
             "match": "*",
             "match_mapping_type": "string",
             "mapping": {
-              "type": "string",
-              "index": "not_analyzed",
+              "type": "keyword",
+              "index": true,
               "omit_norms": true
             }
           }
         }
       ],
       "properties": {
         "@version": {
-          "type": "string",
-          "index": "not_analyzed"
+          "type": "keyword",
+          "index": true
         },
         "@raw": {
-          "type": "string",
-          "index": "analyzed",
+          "type": "text",
+          "index": true,
           "norms": {
             "enabled": false
           }
diff --git a/jobs/ingestor_syslog/spec b/jobs/ingestor_syslog/spec
@@ -16,6 +16,7 @@ templates:
   config/syslog_tls.crt.erb: config/syslog_tls.crt
   config/syslog_tls.key.erb: config/syslog_tls.key
   config/logstash.yml.erb: config/logstash.yml
+  config/jvm.options.erb: config/jvm.options
   data/properties.sh.erb: data/properties.sh
   helpers/ctl_setup.sh: helpers/ctl_setup.sh
   helpers/ctl_utils.sh: helpers/ctl_utils.sh
@@ -39,6 +40,9 @@ properties:
   logstash.heap_percentage:
     description: The percentage value used in the calculation to set the heap size.
     default: 46
+  logstash.jvm_options:
+    description: additional jvm options
+    default: []
   logstash.metadata_level:
     description: "Whether to include additional metadata throughout the event lifecycle. NONE = disabled, DEBUG = fully enabled"
     default: "NONE"
diff --git a/jobs/ingestor_syslog/templates/bin/ingestor_syslog_ctl b/jobs/ingestor_syslog/templates/bin/ingestor_syslog_ctl
@@ -93,6 +93,13 @@ case $1 in
 
     echo "} #close filters" >> ${JOB_DIR}/config/logstash.conf
 
+    # clear persistent queue if the upgrade failed last run
+    if cat $LOG_DIR/$JOB_NAME.stdout.log | grep -a 'QueueUpgrade - Logstash was unable to upgrade your persistent queue data' >/dev/null ; then
+      mkdir ${STORE_DIR}/oldqueue.$$
+      mv ${STORE_DIR}/queue ${STORE_DIR}/.lock ${STORE_DIR}/dead_letter_queue ${STORE_DIR}/uuid ${STORE_DIR}/oldqueue.$$
+      mv $LOG_DIR/$JOB_NAME.stdout.log $LOG_DIR/$JOB_NAME.stdout.log.old
+    fi
+
     exec chpst -u vcap:vcap /var/vcap/packages/logstash/bin/logstash \
         --path.data ${STORE_DIR} \
         --path.config ${JOB_DIR}/config/logstash.conf \
diff --git a/jobs/ingestor_syslog/templates/config/input_and_output.conf.erb b/jobs/ingestor_syslog/templates/config/input_and_output.conf.erb
@@ -57,7 +57,6 @@ output {
               options = {
                 "hosts" => [ p('logstash_parser.elasticsearch.data_hosts').map { |ip| "#{ip}:9200" }.join(',') ],
                 "index" => p('logstash_parser.elasticsearch.index'),
-                "document_type" => p('logstash_parser.elasticsearch.index_type'),
                 "manage_template" => false
               }
               if p('logstash_parser.elasticsearch.idle_flush_time', nil)
diff --git a/jobs/ingestor_syslog/templates/config/jvm.options.erb b/jobs/ingestor_syslog/templates/config/jvm.options.erb
@@ -0,0 +1,4 @@
+# the default /tmp dir is often noexec, so we need to use one that is not.
+-Djava.io.tmpdir=/var/vcap/data/sys/tmp/ingestor_syslog
+
+<% p('logstash.jvm_options').each do |opt| %><%= opt %><% end %>
diff --git a/packages/elasticsearch/packaging b/packages/elasticsearch/packaging
@@ -1,3 +1,3 @@
 set -e
 
-tar xzf elasticsearch/elasticsearch-5.6.13.tar.gz -C $BOSH_INSTALL_TARGET --strip-components 1
+tar xzf elasticsearch/elasticsearch-6.4.3.tar.gz -C $BOSH_INSTALL_TARGET --strip-components 1
diff --git a/packages/elasticsearch/spec b/packages/elasticsearch/spec
@@ -2,4 +2,4 @@
 name: elasticsearch
 
 files:
-  - elasticsearch/elasticsearch-5.6.13.tar.gz
+  - elasticsearch/elasticsearch-6.4.3.tar.gz
diff --git a/packages/java8/packaging b/packages/java8/packaging
@@ -1,3 +1,3 @@
 set -e
 
-tar xzf java8/openjdk-1.8.0_152.tar.gz -C $BOSH_INSTALL_TARGET
+tar xzf java8/openjdk-1.8.0_192.tar.gz -C $BOSH_INSTALL_TARGET
diff --git a/packages/java8/spec b/packages/java8/spec
@@ -2,4 +2,4 @@
 name: java8
 
 files:
-  - java8/openjdk-1.8.0_152.tar.gz
+  - java8/openjdk-1.8.0_192.tar.gz
diff --git a/packages/kibana/packaging b/packages/kibana/packaging
@@ -1,3 +1,3 @@
 set -e
 
-tar xzf kibana/kibana-5.6.13-linux-x86_64.tar.gz -C $BOSH_INSTALL_TARGET --strip-components 1
+tar xzf kibana/kibana-6.4.3-linux-x86_64.tar.gz -C $BOSH_INSTALL_TARGET --strip-components 1
diff --git a/packages/kibana/spec b/packages/kibana/spec
@@ -2,4 +2,4 @@
 name: kibana
 
 files:
-  - kibana/kibana-5.6.13-linux-x86_64.tar.gz
+  - kibana/kibana-6.4.3-linux-x86_64.tar.gz
diff --git a/packages/logstash/packaging b/packages/logstash/packaging
@@ -2,7 +2,7 @@ set -e
 
 export JAVA_HOME=/var/vcap/packages/java8
 
-tar xzf logstash/logstash-5.6.13.tar.gz -C $BOSH_INSTALL_TARGET --strip-components 1
+tar xzf logstash/logstash-6.4.3.tar.gz -C $BOSH_INSTALL_TARGET --strip-components 1
 
 # Installs missing plugins
 $BOSH_INSTALL_TARGET/bin/logstash-plugin install file://$PWD/logstash/logstash-filter-alter-3.0.2.zip
diff --git a/packages/logstash/spec b/packages/logstash/spec
@@ -5,7 +5,7 @@ dependencies:
   - java8
 
 files:
-  - logstash/logstash-5.6.13.tar.gz
+  - logstash/logstash-6.4.3.tar.gz
   - logstash/logstash-filter-alter-3.0.2.zip
   - logstash/logstash-filter-translate-3.0.3.zip
   - logstash/logstash-input-relp-3.0.2.zip
diff --git a/src/logsearch-config/spec/logstash-templates/input_and_output/test_default-expected.conf b/src/logsearch-config/spec/logstash-templates/input_and_output/test_default-expected.conf
diff --git a/src/logsearch-config/spec/logstash-templates/input_and_output/test_enabled_debug-expected.conf b/src/logsearch-config/spec/logstash-templates/input_and_output/test_enabled_debug-expected.conf
diff --git a/src/logsearch-config/spec/logstash-templates/input_and_output/test_inputs-expected.conf b/src/logsearch-config/spec/logstash-templates/input_and_output/test_inputs-expected.conf
diff --git a/src/logsearch-config/spec/logstash-templates/input_and_output/test_outputs-expected.conf b/src/logsearch-config/spec/logstash-templates/input_and_output/test_outputs-expected.conf
diff --git a/src/logsearch-config/spec/logstash-templates/input_and_output/test_outputs_elasticsearch_all_properties-expected.conf b/src/logsearch-config/spec/logstash-templates/input_and_output/test_outputs_elasticsearch_all_properties-expected.conf
diff --git a/src/logsearch-config/spec/logstash-templates/input_and_output/test_outputs_elasticsearch_required_properties-expected.conf b/src/logsearch-config/spec/logstash-templates/input_and_output/test_outputs_elasticsearch_required_properties-expected.conf
