Adding a new "copy" operation

lib/logstash/filters/mutate.rb

@@ -162,6 +162,22 @@ class LogStash::Filters::Mutate < LogStash::Filters::Base
   #     }
   config :merge, :validate => :hash
 
+  # Move all properties of a sub-structure of the event to the `target` field (which is the root level if not specified).
+  # By default, all the other properties of the target are kept, but it is also possible to erase them by setting `empty_target` to `true`
+  #
+  # Example:
+  # [source,ruby]
+  #     filter {
+  #       mutate {
+  #          move => {
+  #             "field" => "moved_field"
+  #             "target" => "target_field"
+  #             "empty_target" => true
+  #          }
+  #       }
+  #     }
+  config :move, :validate => :hash
+
   TRUE_REGEX = (/^(true|t|yes|y|1)$/i).freeze
   FALSE_REGEX = (/^(false|f|no|n|0)$/i).freeze
   CONVERT_PREFIX = "convert_".freeze
@@ -212,6 +228,7 @@ def filter(event)
     split(event) if @split
     join(event) if @join
     merge(event) if @merge
+    move(event) if @move
 
     filter_matched(event)
   end
@@ -427,4 +444,56 @@ def merge(event)
       end
     end
   end
+
+  def move(event)
+    if @move['field'].nil?
+      raise LogStash::ConfigurationError, I18n.t(
+          "logstash.agent.configuration.invalid_plugin_register",
+          :plugin => "filter",
+          :type => "mutate",
+          :error => "No field to move has been specified"
+      )
+    end
+
+    field = event.sprintf(@move['field'])
+    if event.get(field).nil?
+      @logger.warn("No field available in event", :field => field)
+      return
+    end
+
+    unless event.get(field).is_a?(Hash)
+      @logger.warn("Field to move must be a Hash", :field => field, :type => event.get(field).class)
+      return
+    end
+
+    # delete all target fields first?
+    if @move['empty_target'] and convert_boolean(@move['empty_target'])
+      # empty the root?
+      if @move['target'].nil?
+        event.to_hash.each do |k, v|
+          event.remove(k) unless k == field
+        end
+      else
+        # empty the target?
+        event.set(@move['target'], {})
+      end
+    else
+      # make sure that the target is a Hash and not a string, etc
+      unless @move['target'].nil?
+        unless event.get(@move['target']).is_a?(Hash)
+          event.set(@move['target'], {})
+        end
+      end
+    end
+
+    # move sub-fields to target level
+    event.get(field).each do |k, v|
+      target_key = k if @move['target'].nil?
+      target_key = "#{@move['target']}[#{k}]" unless @move['target'].nil?
+      event.set(target_key, v)
+    end
+
+    # delete moved sub-field
+    event.remove(field)
+  end
 end

spec/filters/mutate_spec.rb

@@ -187,7 +187,7 @@ def pattern_path(path)
       config <<-CONFIG
         filter {
           mutate {
-            convert => [ "message", "int"] //should be integer
+            convert => [ "message", "int"] #should be integer
           }
         }
       CONFIG
@@ -638,4 +638,164 @@ def pattern_path(path)
     end
   end
 
+  describe "no sub-field to move" do
+    config '
+      filter {
+        mutate {
+          move => {
+          }
+        }
+      }'
+
+    sample("foo" => "bar") do
+      expect {subject}.to raise_error LogStash::ConfigurationError
+    end
+  end
+
+  describe "move empty sub-field at root level" do
+    config '
+      filter {
+        mutate {
+          move => {
+            field => "sub"
+          }
+        }
+      }'
+
+    sample("foo" => "bar") do
+      expect(subject.get("foo")).to eq "bar"
+    end
+  end
+
+  describe "move non-Hash sub-field at root level" do
+    config '
+      filter {
+        mutate {
+          move => {
+            field => "sub"
+          }
+        }
+      }'
+
+    sample("foo" => "bar", "sub" => "123") do
+      expect(subject.get("foo")).to eq "bar"
+      expect(subject.get("sub")).to eq "123"
+    end
+  end
+
+  describe "move sub-fields at root level" do
+    config '
+      filter {
+        mutate {
+          move => {
+            field => "sub"
+          }
+        }
+      }'
+
+    sample("foo" => "bar", "sub" => { "field1" => "value1", "field2" => "value2"}) do
+      expect(subject.get("foo")).to eq "bar"
+      expect(subject.get("field1")).to eq "value1"
+      expect(subject.get("field2")).to eq "value2"
+      expect(subject.get("sub")).to eq nil
+    end
+  end
+
+  describe "move sub-fields at root level and erase root fields" do
+    config '
+      filter {
+        mutate {
+          move => {
+            field => "sub"
+            empty_target => true
+          }
+        }
+      }'
+
+    sample("foo" => "bar", "sub" => { "field1" => "value1", "field2" => "value2"}) do
+      expect(subject.get("foo")).to eq nil
+      expect(subject.get("field1")).to eq "value1"
+      expect(subject.get("field2")).to eq "value2"
+      expect(subject.get("sub")).to eq nil
+    end
+  end
+
+  describe "move sub-fields to non-existing target" do
+    config '
+      filter {
+        mutate {
+          move => {
+            field => "sub"
+            target => "target"
+          }
+        }
+      }'
+
+    sample("foo" => "bar", "sub" => { "field1" => "value1", "field2" => "value2"}) do
+      expect(subject.get("foo")).to eq "bar"
+      expect(subject.get("[target][field1]")).to eq "value1"
+      expect(subject.get("[target][field2]")).to eq "value2"
+      expect(subject.get("sub")).to eq nil
+    end
+  end
+
+  describe "move sub-fields to existing hash target" do
+    config '
+      filter {
+        mutate {
+          move => {
+            field => "sub"
+            target => "target"
+          }
+        }
+      }'
+
+    sample("foo" => "bar", "sub" => { "field1" => "value1", "field2" => "value2"}, "target" => { "field3" => "value3" }) do
+      expect(subject.get("foo")).to eq "bar"
+      expect(subject.get("[target][field1]")).to eq "value1"
+      expect(subject.get("[target][field2]")).to eq "value2"
+      expect(subject.get("[target][field3]")).to eq "value3"
+      expect(subject.get("sub")).to eq nil
+    end
+  end
+
+  describe "move sub-fields to existing hash target and erase target fields" do
+    config '
+      filter {
+        mutate {
+          move => {
+            field => "sub"
+            target => "target"
+            empty_target => true
+          }
+        }
+      }'
+
+    sample("foo" => "bar", "sub" => { "field1" => "value1", "field2" => "value2"}, "target" => { "field3" => "value3" }) do
+      expect(subject.get("foo")).to eq "bar"
+      expect(subject.get("[target][field1]")).to eq "value1"
+      expect(subject.get("[target][field2]")).to eq "value2"
+      expect(subject.get("[target][field3]")).to eq nil
+      expect(subject.get("sub")).to eq nil
+    end
+  end
+
+  describe "move sub-fields to existing non-hash target" do
+    config '
+      filter {
+        mutate {
+          move => {
+            field => "sub"
+            target => "target"
+          }
+        }
+      }'
+
+    sample("foo" => "bar", "sub" => { "field1" => "value1", "field2" => "value2"}, "target" => "123" ) do
+      expect(subject.get("foo")).to eq "bar"
+      expect(subject.get("[target][field1]")).to eq "value1"
+      expect(subject.get("[target][field2]")).to eq "value2"
+      expect(subject.get("sub")).to eq nil
+    end
+  end
 end