Merge branch 'master' into hiera-plugindir-call

Author: tylerjl

CHANGELOG.md

@@ -9,6 +9,7 @@ Puppet 4.10.0 is the new minimum required version of Puppet.
 #### Fixes
 * The Elasticsearch log directory is no longer recursively managed to avoid stomping on the user/mode settings that Elasticsearch prefers.
 * Package management on apt-based systems no longer encounters dependency errors when `manage_repo => false`.
+* Plugin configuration files are now more well-supported by respecting subdirectory copy recursion. See `elasticsearch::configdir_recurselimit` for more information.
 * An error related to elasticsearch_roles and `yield` errors has been fixed
 * Correctly permit instances to be set to `absent` without errors.
 * Resolved an issue arising from the use of `hiera()` in Hiera yaml data files.

CONTRIBUTORS

@@ -32,3 +32,4 @@ Jose Luis Ledesma (sp-joseluis-ledesma)
 Matthias Baur (baurmatt)
 Gavin Williams (fatmcgav)
 Giedrius Statkevičius (GiedriusS)
+Jakub Pieńkowski (Jakski)

data/common.yaml

@@ -11,6 +11,7 @@ elasticsearch::api_timeout: 10
 elasticsearch::autoupgrade: false
 elasticsearch::config: {}
 elasticsearch::configdir: /etc/elasticsearch
+elasticsearch::configdir_recurselimit: 2
 elasticsearch::daily_rolling_date_pattern: |
   "'.'yyyy-MM-dd"
 elasticsearch::datadir_instance_directories: true

lib/facter/es_facts.rb

@@ -3,6 +3,7 @@
 require 'yaml'
 
 # Helper module to encapsulate custom fact injection
+# rubocop:disable Metrics/ModuleLength
 module EsFacts
   # Add a fact to the catalog of host facts
   def self.add_fact(prefix, key, value)
@@ -19,18 +20,18 @@ def self.ssl?(config)
       'searchguard.ssl.http.enabled'
     ]
 
-    tls_keys.any? { |key| config.key? key and config[key] == true }
+    tls_keys.any? { |key| (config.key? key) && (config[key] == true) }
   end
 
-  # Helper to determine the instance port number
-  def self.get_port(config)
+  # Helper to determine the instance http.port number
+  def self.get_httpport(config)
     enabled = 'http.enabled'
-    port = 'http.port'
+    httpport = 'http.port'
 
-    if not config[enabled].nil? and config[enabled] == 'false'
+    if !config[enabled].nil? && config[enabled] == 'false'
       false
-    elsif not config[port].nil?
-      { config[port] => ssl?(config) }
+    elsif !config[httpport].nil?
+      { config[httpport] => ssl?(config) }
     else
       { '9200' => ssl?(config) }
     end
@@ -44,8 +45,13 @@ def self.get_port(config)
   # rubocop:disable Metrics/PerceivedComplexity
   def self.run
     dir_prefix = '/etc/elasticsearch'
-    # Ports is a hash of port_number => ssl?
-    ports = {}
+    # httpports is a hash of port_number => ssl?
+    httpports = {}
+    transportports = []
+    http_bound_addresses = []
+    transport_bound_addresses = []
+    transport_publish_addresses = []
+    nodes = {}
 
     # only when the directory exists we need to process the stuff
     return unless File.directory?(dir_prefix)
@@ -55,22 +61,22 @@ def self.run
 
       if File.readable?("#{dir_prefix}/#{dir}/elasticsearch.yml")
         config_data = YAML.load_file("#{dir_prefix}/#{dir}/elasticsearch.yml")
-        port = get_port(config_data)
-        next unless port
-        ports.merge! port
+        httpport = get_httpport(config_data)
+        httpports.merge! httpport if httpport
       end
     end
 
     begin
-      if ports.keys.count > 0
+      if httpports.keys.count > 0
 
-        add_fact('elasticsearch', 'ports', ports.keys.join(','))
-        ports.each_pair do |port, ssl|
+        add_fact('elasticsearch', 'ports', httpports.keys.join(','))
+
+        httpports.each_pair do |httpport, ssl|
           next if ssl
 
-          key_prefix = "elasticsearch_#{port}"
+          key_prefix = "elasticsearch_#{httpport}"
 
-          uri = URI("http://localhost:#{port}")
+          uri = URI("http://localhost:#{httpport}")
           http = Net::HTTP.new(uri.host, uri.port)
           http.read_timeout = 10
           http.open_timeout = 2
@@ -81,7 +87,7 @@ def self.run
           add_fact(key_prefix, 'name', json_data['name'])
           add_fact(key_prefix, 'version', json_data['version']['number'])
 
-          uri2 = URI("http://localhost:#{port}/_nodes/#{json_data['name']}")
+          uri2 = URI("http://localhost:#{httpport}/_nodes/#{json_data['name']}")
           http2 = Net::HTTP.new(uri2.host, uri2.port)
           http2.read_timeout = 10
           http2.open_timeout = 2
@@ -110,10 +116,29 @@ def self.run
             end
           end
           add_fact(key_prefix, 'plugins', plugin_names.join(','))
+
+          nodes_data['http']['bound_address'].each { |i| http_bound_addresses << i }
+          nodes_data['transport']['bound_address'].each { |i| transport_bound_addresses << i }
+          transport_publish_addresses << nodes_data['transport']['publish_address'] unless nodes_data['transport']['publish_address'].nil?
+          transportports << nodes_data['settings']['transport']['tcp']['port'] unless nodes_data['settings']['transport']['tcp'].nil? or nodes_data['settings']['transport']['tcp']['port'].nil?
+
+          node = { 'http_ports' => httpports.keys,
+                   'transport_ports' => transportports,
+                   'http_bound_addresses' => http_bound_addresses,
+                   'transport_bound_addresses' => transport_bound_addresses,
+                   'transport_publish_addresses' => transport_publish_addresses,
+                   json_data['name'] => { 'settings' => nodes_data['settings'], 'http' => nodes_data['http'], 'transport' => nodes_data['transport'] } }
+          nodes.merge! node
         end
       end
     rescue
     end
+    Facter.add(:elasticsearch) do
+      setcode do
+        nodes
+      end
+      nodes unless nodes.empty?
+    end
   end
   # rubocop:enable Metrics/CyclomaticComplexity
   # rubocop:enable Metrics/PerceivedComplexity

manifests/init.pp

@@ -65,6 +65,10 @@
 #   Directory containing the elasticsearch configuration.
 #   Use this setting if your packages deviate from the norm (`/etc/elasticsearch`)
 #
+# @param configdir_recurselimit
+#   Dictates how deeply the file copy recursion logic should descend when
+#   copying files from the `configdir` to instance `configdir`s.
+#
 # @param daily_rolling_date_pattern
 #   File pattern for the file appender log when file_rolling_type is 'dailyRollingFile'.
 #
@@ -309,6 +313,7 @@
   Boolean                                         $autoupgrade,
   Hash                                            $config,
   Stdlib::Absolutepath                            $configdir,
+  Integer                                         $configdir_recurselimit,
   String                                          $daily_rolling_date_pattern,
   Elasticsearch::Multipath                        $datadir,
   Boolean                                         $datadir_instance_directories,
@@ -585,4 +590,8 @@
   -> Elasticsearch::Instance <| ensure == 'absent' |>
   Elasticsearch::Snapshot_repository <| |>
   -> Elasticsearch::Instance <| ensure == 'absent' |>
+
+  # Ensure scripts are installed before copying them to configuration directory
+  Elasticsearch::Script <| |>
+  -> File["${configdir}/scripts"]
 }

manifests/instance.pp

@@ -23,6 +23,10 @@
 #   Path to directory containing the elasticsearch configuration.
 #   Use this setting if your packages deviate from the norm (/etc/elasticsearch).
 #
+# @param configdir_recurselimit
+#   Dictates how deeply the file copy recursion logic should descend when
+#   copying files from the `elasticsearch::configdir` to instance `configdir`s.
+#
 # @param daily_rolling_date_pattern
 #   File pattern for the file appender log when file_rolling_type is `dailyRollingFile`
 #
@@ -130,6 +134,7 @@
   Optional[Stdlib::Absolutepath]     $certificate                   = undef,
   Optional[Hash]                     $config                        = undef,
   Stdlib::Absolutepath               $configdir                     = "${elasticsearch::configdir}/${name}",
+  Integer                            $configdir_recurselimit        = $elasticsearch::configdir_recurselimit,
   String                             $daily_rolling_date_pattern    = $elasticsearch::daily_rolling_date_pattern,
   Optional[Elasticsearch::Multipath] $datadir                       = undef,
   Boolean                            $datadir_instance_directories  = $elasticsearch::datadir_instance_directories,
@@ -366,7 +371,7 @@
         "${elasticsearch::configdir}/log4j2.properties",
       ],
       recurse      => 'remote',
-      recurselimit => 1,
+      recurselimit => $configdir_recurselimit,
       source       => $elasticsearch::configdir,
       purge        => $elasticsearch::purge_configdir,
       force        => $elasticsearch::purge_configdir,

spec/defines/005_elasticsearch_instance_spec.rb

@@ -799,7 +799,6 @@ class { 'elasticsearch':
                 -Dlog4j2.disable.jmx=true.
                 -XX:\+AlwaysPreTouch.
                 -XX:\+HeapDumpOnOutOfMemoryError.
-                -XX:\+PrintGCApplicationStoppedTime.
                 -XX:\+PrintGCDateStamps.
                 -XX:\+PrintGCDetails.
                 -XX:\+PrintTenuringDistribution.
@@ -842,7 +841,6 @@ class { 'elasticsearch':
                 -Dlog4j2.disable.jmx=true.
                 -XX:\+AlwaysPreTouch.
                 -XX:\+HeapDumpOnOutOfMemoryError.
-                -XX:\+PrintGCApplicationStoppedTime.
                 -XX:\+PrintGCDateStamps.
                 -XX:\+PrintGCDetails.
                 -XX:\+PrintTenuringDistribution.

templates/etc/elasticsearch/jvm.options.erb

@@ -28,7 +28,6 @@ defaults = {
   'PrintGCDetails' => '-XX:+PrintGCDetails',
   'PrintGCDateStamps' => '-XX:+PrintGCDateStamps',
   'PrintTenuringDistribution' => '-XX:+PrintTenuringDistribution',
-  'PrintGCApplicationStoppedTime' => '-XX:+PrintGCApplicationStoppedTime',
   'Xloggc' => "-Xloggc:#{@logdir}/gc.log",
   'UseGCLogFileRotation' => '-XX:+UseGCLogFileRotation',
   'NumberOfGCLogFiles' => '-XX:NumberOfGCLogFiles=32',