Add flag to enable disabling of 'host' field auto-population

Add `add_hostname` flag - when enabled, the `host` field will be populated
by the value supplied by the beat in the `hostname` field if that field is present.

The flag defaults to true to retain backwards compatibility with earlier versions of
the beats input.

The field is intentionally added as a deprecated field - auto populating the `host` field in
this way will be removed in future versions of the beats input.

Fixes #340

Author: robbavey

docs/index.asciidoc

@@ -90,6 +90,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 [cols="<,<,<",options="header",]
 |=======================================================================
 |Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-add_hostname>> |<<boolean, boolean>>|No
 | <<plugins-{type}s-{plugin}-cipher_suites>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-client_inactivity_timeout>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-host>> |<<string,string>>|No
@@ -112,6 +113,18 @@ input plugins.
 
 &nbsp;
 
+[id="plugins-{type}s-{plugin}-add_hostname"]
+===== `add_hostname`
+
+  added[5.2.0, Field was added to allow users to not automatically add the `host` field in events.]
+  deprecated[5.2.0, In future versions of this plugin, this setting will be removed, and the 'hosts' field will not be added to events.]
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+Flag to determine whether to add `host` field to event using the value supplied by the beat in the `hostname` field.
+
+
 [id="plugins-{type}s-{plugin}-cipher_suites"]
 ===== `cipher_suites`
 

lib/logstash/inputs/beats.rb

@@ -82,6 +82,10 @@ class LogStash::Inputs::Beats < LogStash::Inputs::Base
   # 
   config :ssl_certificate_authorities, :validate => :array, :default => []
 
+  # Flag to determine whether to add host information (provided by the beat in the 'hostname' field) to the event
+  config :add_hostname, :validate => :boolean, :default => true, :deprecated => 'Host field will not be automatically populated by future version of the Beats input'
+
+
   # By default the server doesn't do any client verification.
   # 
   # `peer` will make the server ask the client to provide a certificate. 

lib/logstash/inputs/beats/event_transform_common.rb

@@ -12,6 +12,7 @@ def initialize(input)
     # Copies the beat.hostname field into the host field unless
     # the host field is already defined
     def copy_beat_hostname(event)
+      return unless @input.add_hostname
       host = event.get("[beat][hostname]")
 
       if host && event.get("host").nil?

spec/support/shared_examples.rb

@@ -25,31 +25,126 @@
     expect(subject.get("tags")).to include(tag)
   end
 
-  context "when the `beats.hostname` doesnt exist on the event" do
-    let(:already_exist) { "already_exist" }
-    let(:event_map) { super.merge({ "host" => already_exist }) }
+  context 'when add_hostname is true' do
+    let(:config) { super.merge({'add_hostname' => true})}
 
-    it "doesnt change the value" do
-      expect(subject.get("host")).to eq(already_exist)
+    context 'when a host is provided in beat.host.name' do
+      let(:already_exist) { "already_exist" }
+      let(:producer_host) { "newhost01" }
+      let(:event_map) { super.merge({ "beat" => { "host" => {"name" => producer_host }}}) }
+
+      context "when no `host` key already exists on the event" do
+        it "does not set the host value" do
+          expect(subject.get("host")).to be_nil
+        end
+      end
+
+      context "when `host` key exists on the event" do
+        let(:already_exist) { "already_exist" }
+        let(:event_map) { super.merge({ "host" => already_exist }) }
+
+        it "doesn't override it" do
+          expect(subject.get("host")).to eq(already_exist)
+        end
+      end
     end
-  end
 
-  context "when the `beat.hostname` exist in the event" do
-    let(:producer_host) { "newhost01" }
-    let(:event_map) { super.merge({ "beat" => { "hostname" => producer_host }}) }
+    context "when a host is set in `beat.hostname`" do
+      let(:producer_host) { "newhost01" }
+      let(:event_map) { super.merge({ "beat" => { "hostname" => producer_host }}) }
 
-    context "when `host` key doesn't exist on the event" do
-      it "copy the `beat.hostname` to `host` or backward compatibility" do
-        expect(subject.get("host")).to eq(producer_host)
+      context "when no `host` key already exists on the event" do
+        it "copies the value in `beat.hostname` to `host`" do
+          expect(subject.get("host")).to eq(producer_host)
+        end
+      end
+
+      context "when `host` key exists on the event" do
+        let(:already_exist) { "already_exist" }
+        let(:event_map) { super.merge({ "host" => already_exist }) }
+
+        it "doesn't override it" do
+          expect(subject.get("host")).to eq(already_exist)
+        end
+      end
+    end
+
+    context "when no host is provided in beat" do
+      context "when no `host` key already exists on the event" do
+        it "does not set the host" do
+          expect(subject.get("host")).to be_nil
+        end
+      end
+
+      context "when `host` key already exists on the event" do
+        let(:already_exist) { "already_exist" }
+        let(:event_map) { super.merge({ "host" => already_exist }) }
+
+        it "doesn't override it" do
+          expect(subject.get("host")).to eq(already_exist)
+        end
       end
     end
+  end
 
-    context "when `host` key exists on the event" do
+  context 'when add hostname is false' do
+    let(:config) { super.merge({'add_hostname' => false})}
+
+    context 'when a host is provided in beat.host.name' do
       let(:already_exist) { "already_exist" }
-      let(:event_map) { super.merge({ "host" => already_exist }) }
+      let(:producer_host) { "newhost01" }
+      let(:event_map) { super.merge({ "beat" => { "host" => {"name" => producer_host }}}) }
+
+      context "when no `host` key already exists on the event" do
+        it "does not set the host" do
+          expect(subject.get("host")).to be_nil
+        end
+      end
+
+      context "when `host` key already exists on the event" do
+        let(:already_exist) { "already_exist" }
+        let(:event_map) { super.merge({ "host" => already_exist }) }
+
+        it "doesn't override it" do
+          expect(subject.get("host")).to eq(already_exist)
+        end
+      end
+    end
+
+    context "when a host is provided in `beat.hostname`" do
+      let(:producer_host) { "newhost01" }
+      let(:event_map) { super.merge({ "beat" => { "hostname" => producer_host }}) }
+
+      context "when no `host` key already exists on the event" do
+        it "does not set the host" do
+          expect(subject.get("host")).to be_nil
+        end
+      end
+
+      context "when `host` key already exists on the event" do
+        let(:already_exist) { "already_exist" }
+        let(:event_map) { super.merge({ "host" => already_exist }) }
+
+        it "doesn't override it" do
+          expect(subject.get("host")).to eq(already_exist)
+        end
+      end
+    end
+
+    context "when no host is provided in beat" do
+      context "when no `host` key already exists on the event" do
+        it "does not set the host" do
+          expect(subject.get("host")).to be_nil
+        end
+      end
+
+      context "when `host` key already exists on the event" do
+        let(:already_exist) { "already_exist" }
+        let(:event_map) { super.merge({ "host" => already_exist }) }
 
-      it "doesn't override it" do
-        expect(subject.get("host")).to eq(already_exist)
+        it "doesn't override it" do
+          expect(subject.get("host")).to eq(already_exist)
+        end
       end
     end
   end