From b3b49b6086098bf4db3a954bf15b55a7079f934e Mon Sep 17 00:00:00 2001 From: gferreux Date: Mon, 4 Mar 2019 14:24:23 +0100 Subject: [PATCH 1/6] add omitempty function --- lib/logstash/filters/mutate.rb | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/lib/logstash/filters/mutate.rb b/lib/logstash/filters/mutate.rb index f94f63b..6a7ceab 100644 --- a/lib/logstash/filters/mutate.rb +++ b/lib/logstash/filters/mutate.rb @@ -207,6 +207,17 @@ class LogStash::Filters::Mutate < LogStash::Filters::Base # } config :copy, :validate => :hash + # Omit a variable if it as a default value. + # + # Example: + # [source,ruby] + # filter { + # mutate { + # omitempty => [ "fieldname" ] + # } + # } + config :omitempty, :validate => :array + TRUE_REGEX = (/^(true|t|yes|y|1|1.0)$/i).freeze FALSE_REGEX = (/^(false|f|no|n|0|0.0)$/i).freeze CONVERT_PREFIX = "convert_".freeze @@ -260,6 +271,7 @@ def filter(event) join(event) if @join merge(event) if @merge copy(event) if @copy + omitempty(event) if @omitempty filter_matched(event) end @@ -466,6 +478,26 @@ def capitalize(event) end end + def omitempty(event) + @omitempty.each do |field| + original = event.get(field) + if original.nil? + event.remove(field) + next + end + result = case original + when String, Array, Hash + original.empty? ? nil : original + when Integer + original == 0 ? nil : original + else + @logger.debug? && @logger.debug("Can't omitempty something that isn't a string,array,hash,integer", :field => field, :value => original) + original + end + event.remove(field) if result.nil? + end + end + def split(event) @split.each do |field, separator| value = event.get(field) From 547f54c9cab95cbd4e73c7f2075e25a8df17af40 Mon Sep 17 00:00:00 2001 From: gferreux Date: Mon, 4 Mar 2019 14:25:35 +0100 Subject: [PATCH 2/6] bump version --- logstash-filter-mutate.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logstash-filter-mutate.gemspec b/logstash-filter-mutate.gemspec index e2fbc0a..e3c5400 100644 --- a/logstash-filter-mutate.gemspec +++ b/logstash-filter-mutate.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-filter-mutate' - s.version = '3.4.0' + s.version = '3.4.1' s.licenses = ['Apache License (2.0)'] s.summary = "Performs mutations on fields" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From a74db48e8cb5ff4491ccd20ba8e615a632704da0 Mon Sep 17 00:00:00 2001 From: gferreux Date: Mon, 4 Mar 2019 14:26:43 +0100 Subject: [PATCH 3/6] add myself to contributors --- CONTRIBUTORS | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS b/CONTRIBUTORS index f99fcce..2d72da2 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -25,6 +25,7 @@ Contributors: * Tal Levy (talevy) * piavlo * Abdul Haseeb Hussain (AbdulHaseebHussain) +* Gregory Ferreux (gferreux) Note: If you've sent us patches, bug reports, or otherwise contributed to Logstash, and you aren't on the list above and want to be, please let us know From 5aaf39ac947f5bfa2fd9cad3b81a96b7a9c410ba Mon Sep 17 00:00:00 2001 From: gferreux Date: Mon, 4 Mar 2019 14:28:38 +0100 Subject: [PATCH 4/6] update CHANGELOG --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b8ed17..968809e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 3.4.1 + - feature: Added 'omitempty' feature to remove fields with default or null values + ## 3.4.0 - Added ability to directly convert from integer and float to boolean [#127](https://github.com/logstash-plugins/logstash-filter-mutate/pull/127) From 38a7eb562fb3df51c9e22ca931a7a3def9774c04 Mon Sep 17 00:00:00 2001 From: gferreux Date: Mon, 4 Mar 2019 16:49:37 +0100 Subject: [PATCH 5/6] add tests --- spec/filters/mutate_spec.rb | 41 +++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/spec/filters/mutate_spec.rb b/spec/filters/mutate_spec.rb index a1744e5..3c57dc4 100644 --- a/spec/filters/mutate_spec.rb +++ b/spec/filters/mutate_spec.rb @@ -1038,4 +1038,45 @@ def pattern_path(path) end end + describe "omitempty" do + + config <<-CONFIG + filter { + mutate { + omitempty => ["field"] + } + } + CONFIG + + context 'when field is a string' do + sample({'field' => ''}) do + expect(subject).not_to include("field") + end + end + + context 'when field is an integer' do + sample({'field' => 0}) do + expect(subject).not_to include("field") + end + end + + context 'when field is an empty hash' do + sample({'field' => {}}) do + expect(subject).not_to include("field") + end + end + + context 'when field is an empty array' do + sample({'field' => []}) do + expect(subject).not_to include("field") + end + end + + context 'when field is null' do + sample({'field' => nil}) do + expect(subject).not_to include("field") + end + end + end + end From 6bf4ad93b7d14ca6112d746f89c1cdc68f5d141f Mon Sep 17 00:00:00 2001 From: Greg Date: Tue, 7 Jan 2020 15:09:19 +0100 Subject: [PATCH 6/6] Update CHANGELOG.md resolve conflict with 3.5.0 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 968809e..b62e4d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## 3.4.1 +## 3.5.1 - feature: Added 'omitempty' feature to remove fields with default or null values ## 3.4.0