From cb2af59ee09d56019ea20f17db6a087fe233d48f Mon Sep 17 00:00:00 2001 From: val Date: Wed, 15 Feb 2017 11:11:19 +0100 Subject: [PATCH 1/4] First iteration at adding a new "copy" operation --- lib/logstash/filters/mutate.rb | 55 ++++++++++++++++++++++- spec/filters/mutate_spec.rb | 81 ++++++++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+), 1 deletion(-) diff --git a/lib/logstash/filters/mutate.rb b/lib/logstash/filters/mutate.rb index a051420..7b70d0a 100644 --- a/lib/logstash/filters/mutate.rb +++ b/lib/logstash/filters/mutate.rb @@ -112,7 +112,7 @@ class LogStash::Filters::Mutate < LogStash::Filters::Base # } config :lowercase, :validate => :array - # Split a field to an array using a separator character. Only works on string + # copy['field'] a field to an array using a separator character. Only works on string # fields. # # Example: @@ -162,6 +162,21 @@ class LogStash::Filters::Mutate < LogStash::Filters::Base # } config :merge, :validate => :hash + # Copy all properties in a sub-structure of the event to the root level. By default, all the other root level + # properties are kept, but it is also possible to erase them by setting `empty_root` to `true` + # + # Example: + # [source,ruby] + # filter { + # mutate { + # copy => { + # "field" => "copied_sub_field" + # "empty_root" => true + # } + # } + # } + config :copy, :validate => :hash + TRUE_REGEX = (/^(true|t|yes|y|1)$/i).freeze FALSE_REGEX = (/^(false|f|no|n|0)$/i).freeze CONVERT_PREFIX = "convert_".freeze @@ -212,6 +227,7 @@ def filter(event) split(event) if @split join(event) if @join merge(event) if @merge + copy(event) if @copy filter_matched(event) end @@ -427,4 +443,41 @@ def merge(event) end end end + + def copy(event) + if @copy['field'].nil? + raise LogStash::ConfigurationError, I18n.t( + "logstash.agent.configuration.invalid_plugin_register", + :plugin => "filter", + :type => "mutate", + :error => "No field to copy has been specified" + ) + end + + field = event.sprintf(@copy['field']) + if event.get(field).nil? + @logger.warn("No field available in event", :field => field) + return + end + + unless event.get(field).is_a?(Hash) + @logger.warn("Field to copy must be a Hash", :field => field, :type => event.get(field).class) + return + end + + # delete all root fields first + if @copy['empty_root'] and convert_boolean(@copy['empty_root']) + event.to_hash.each do |k, v| + event.remove(k) unless k == field + end + end + + # copy sub-fields to root level + event.get(field).each do |k, v| + event.set(k, v) + end + + # delete copied sub-field + event.remove(field) + end end diff --git a/spec/filters/mutate_spec.rb b/spec/filters/mutate_spec.rb index 470034b..68ed8df 100644 --- a/spec/filters/mutate_spec.rb +++ b/spec/filters/mutate_spec.rb @@ -638,4 +638,85 @@ def pattern_path(path) end end + describe "no sub-field to copy" do + config ' + filter { + mutate { + copy => { + } + } + }' + + sample("foo" => "bar") do + expect {subject}.to raise_error LogStash::ConfigurationError + end + end + + describe "copy empty sub-field at root level" do + config ' + filter { + mutate { + copy => { + field => "sub" + } + } + }' + + sample("foo" => "bar") do + expect(subject.get("foo")).to eq "bar" + end + end + + describe "copy non-Hash sub-field at root level" do + config ' + filter { + mutate { + copy => { + field => "sub" + } + } + }' + + sample("foo" => "bar", "sub" => "123") do + expect(subject.get("foo")).to eq "bar" + expect(subject.get("sub")).to eq "123" + end + end + + describe "copy sub-fields at root level" do + config ' + filter { + mutate { + copy => { + field => "sub" + } + } + }' + + sample("foo" => "bar", "sub" => { "field1" => "value1", "field2" => "value2"}) do + expect(subject.get("foo")).to eq "bar" + expect(subject.get("field1")).to eq "value1" + expect(subject.get("field2")).to eq "value2" + expect(subject.get("sub")).to eq nil + end + end + + describe "copy sub-fields at root level and erase root fields" do + config ' + filter { + mutate { + copy => { + field => "sub" + empty_root => true + } + } + }' + + sample("foo" => "bar", "sub" => { "field1" => "value1", "field2" => "value2"}) do + expect(subject.get("foo")).to eq nil + expect(subject.get("field1")).to eq "value1" + expect(subject.get("field2")).to eq "value2" + expect(subject.get("sub")).to eq nil + end + end end From 67211056e5e2546aaba0ff2fe2283fe00e6b1f58 Mon Sep 17 00:00:00 2001 From: val Date: Wed, 15 Feb 2017 11:12:19 +0100 Subject: [PATCH 2/4] Fix bad copy paste --- lib/logstash/filters/mutate.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/logstash/filters/mutate.rb b/lib/logstash/filters/mutate.rb index 7b70d0a..8483ccb 100644 --- a/lib/logstash/filters/mutate.rb +++ b/lib/logstash/filters/mutate.rb @@ -112,7 +112,7 @@ class LogStash::Filters::Mutate < LogStash::Filters::Base # } config :lowercase, :validate => :array - # copy['field'] a field to an array using a separator character. Only works on string + # Split a field to an array using a separator character. Only works on string # fields. # # Example: From 8324fda5d4f2d157c7eff757f0e72c989ec5ee2a Mon Sep 17 00:00:00 2001 From: val Date: Wed, 15 Feb 2017 11:33:05 +0100 Subject: [PATCH 3/4] Fixed bad comment character in mutate/convert test case config --- spec/filters/mutate_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/filters/mutate_spec.rb b/spec/filters/mutate_spec.rb index 68ed8df..70c0976 100644 --- a/spec/filters/mutate_spec.rb +++ b/spec/filters/mutate_spec.rb @@ -187,7 +187,7 @@ def pattern_path(path) config <<-CONFIG filter { mutate { - convert => [ "message", "int"] //should be integer + convert => [ "message", "int"] #should be integer } } CONFIG From 70b89d4a856ee8bd8edd3915330d6b01b6fe1ea6 Mon Sep 17 00:00:00 2001 From: val Date: Thu, 16 Feb 2017 14:33:15 +0100 Subject: [PATCH 4/4] Renamed action to "move" and added "target" parameter --- lib/logstash/filters/mutate.rb | 54 +++++++++++------- spec/filters/mutate_spec.rb | 101 +++++++++++++++++++++++++++++---- 2 files changed, 125 insertions(+), 30 deletions(-) diff --git a/lib/logstash/filters/mutate.rb b/lib/logstash/filters/mutate.rb index 8483ccb..cd12870 100644 --- a/lib/logstash/filters/mutate.rb +++ b/lib/logstash/filters/mutate.rb @@ -162,20 +162,21 @@ class LogStash::Filters::Mutate < LogStash::Filters::Base # } config :merge, :validate => :hash - # Copy all properties in a sub-structure of the event to the root level. By default, all the other root level - # properties are kept, but it is also possible to erase them by setting `empty_root` to `true` + # Move all properties of a sub-structure of the event to the `target` field (which is the root level if not specified). + # By default, all the other properties of the target are kept, but it is also possible to erase them by setting `empty_target` to `true` # # Example: # [source,ruby] # filter { # mutate { - # copy => { - # "field" => "copied_sub_field" - # "empty_root" => true + # move => { + # "field" => "moved_field" + # "target" => "target_field" + # "empty_target" => true # } # } # } - config :copy, :validate => :hash + config :move, :validate => :hash TRUE_REGEX = (/^(true|t|yes|y|1)$/i).freeze FALSE_REGEX = (/^(false|f|no|n|0)$/i).freeze @@ -227,7 +228,7 @@ def filter(event) split(event) if @split join(event) if @join merge(event) if @merge - copy(event) if @copy + move(event) if @move filter_matched(event) end @@ -444,40 +445,55 @@ def merge(event) end end - def copy(event) - if @copy['field'].nil? + def move(event) + if @move['field'].nil? raise LogStash::ConfigurationError, I18n.t( "logstash.agent.configuration.invalid_plugin_register", :plugin => "filter", :type => "mutate", - :error => "No field to copy has been specified" + :error => "No field to move has been specified" ) end - field = event.sprintf(@copy['field']) + field = event.sprintf(@move['field']) if event.get(field).nil? @logger.warn("No field available in event", :field => field) return end unless event.get(field).is_a?(Hash) - @logger.warn("Field to copy must be a Hash", :field => field, :type => event.get(field).class) + @logger.warn("Field to move must be a Hash", :field => field, :type => event.get(field).class) return end - # delete all root fields first - if @copy['empty_root'] and convert_boolean(@copy['empty_root']) - event.to_hash.each do |k, v| - event.remove(k) unless k == field + # delete all target fields first? + if @move['empty_target'] and convert_boolean(@move['empty_target']) + # empty the root? + if @move['target'].nil? + event.to_hash.each do |k, v| + event.remove(k) unless k == field + end + else + # empty the target? + event.set(@move['target'], {}) + end + else + # make sure that the target is a Hash and not a string, etc + unless @move['target'].nil? + unless event.get(@move['target']).is_a?(Hash) + event.set(@move['target'], {}) + end end end - # copy sub-fields to root level + # move sub-fields to target level event.get(field).each do |k, v| - event.set(k, v) + target_key = k if @move['target'].nil? + target_key = "#{@move['target']}[#{k}]" unless @move['target'].nil? + event.set(target_key, v) end - # delete copied sub-field + # delete moved sub-field event.remove(field) end end diff --git a/spec/filters/mutate_spec.rb b/spec/filters/mutate_spec.rb index 70c0976..ae552a8 100644 --- a/spec/filters/mutate_spec.rb +++ b/spec/filters/mutate_spec.rb @@ -638,11 +638,11 @@ def pattern_path(path) end end - describe "no sub-field to copy" do + describe "no sub-field to move" do config ' filter { mutate { - copy => { + move => { } } }' @@ -652,11 +652,11 @@ def pattern_path(path) end end - describe "copy empty sub-field at root level" do + describe "move empty sub-field at root level" do config ' filter { mutate { - copy => { + move => { field => "sub" } } @@ -667,11 +667,11 @@ def pattern_path(path) end end - describe "copy non-Hash sub-field at root level" do + describe "move non-Hash sub-field at root level" do config ' filter { mutate { - copy => { + move => { field => "sub" } } @@ -683,11 +683,11 @@ def pattern_path(path) end end - describe "copy sub-fields at root level" do + describe "move sub-fields at root level" do config ' filter { mutate { - copy => { + move => { field => "sub" } } @@ -701,13 +701,13 @@ def pattern_path(path) end end - describe "copy sub-fields at root level and erase root fields" do + describe "move sub-fields at root level and erase root fields" do config ' filter { mutate { - copy => { + move => { field => "sub" - empty_root => true + empty_target => true } } }' @@ -719,4 +719,83 @@ def pattern_path(path) expect(subject.get("sub")).to eq nil end end + + describe "move sub-fields to non-existing target" do + config ' + filter { + mutate { + move => { + field => "sub" + target => "target" + } + } + }' + + sample("foo" => "bar", "sub" => { "field1" => "value1", "field2" => "value2"}) do + expect(subject.get("foo")).to eq "bar" + expect(subject.get("[target][field1]")).to eq "value1" + expect(subject.get("[target][field2]")).to eq "value2" + expect(subject.get("sub")).to eq nil + end + end + + describe "move sub-fields to existing hash target" do + config ' + filter { + mutate { + move => { + field => "sub" + target => "target" + } + } + }' + + sample("foo" => "bar", "sub" => { "field1" => "value1", "field2" => "value2"}, "target" => { "field3" => "value3" }) do + expect(subject.get("foo")).to eq "bar" + expect(subject.get("[target][field1]")).to eq "value1" + expect(subject.get("[target][field2]")).to eq "value2" + expect(subject.get("[target][field3]")).to eq "value3" + expect(subject.get("sub")).to eq nil + end + end + + describe "move sub-fields to existing hash target and erase target fields" do + config ' + filter { + mutate { + move => { + field => "sub" + target => "target" + empty_target => true + } + } + }' + + sample("foo" => "bar", "sub" => { "field1" => "value1", "field2" => "value2"}, "target" => { "field3" => "value3" }) do + expect(subject.get("foo")).to eq "bar" + expect(subject.get("[target][field1]")).to eq "value1" + expect(subject.get("[target][field2]")).to eq "value2" + expect(subject.get("[target][field3]")).to eq nil + expect(subject.get("sub")).to eq nil + end + end + + describe "move sub-fields to existing non-hash target" do + config ' + filter { + mutate { + move => { + field => "sub" + target => "target" + } + } + }' + + sample("foo" => "bar", "sub" => { "field1" => "value1", "field2" => "value2"}, "target" => "123" ) do + expect(subject.get("foo")).to eq "bar" + expect(subject.get("[target][field1]")).to eq "value1" + expect(subject.get("[target][field2]")).to eq "value2" + expect(subject.get("sub")).to eq nil + end + end end