From 6e045d6d82b395cdbec40d6fa67ff50f53b77620 Mon Sep 17 00:00:00 2001 From: rkbennett <44292326+rkbennett@users.noreply.github.com> Date: Tue, 2 Feb 2021 22:34:52 -0600 Subject: [PATCH 1/3] Update cidr.rb --- lib/logstash/filters/cidr.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/logstash/filters/cidr.rb b/lib/logstash/filters/cidr.rb index 3bb19b0..973a72d 100644 --- a/lib/logstash/filters/cidr.rb +++ b/lib/logstash/filters/cidr.rb @@ -163,7 +163,8 @@ def filter(event) end end end - + # Sort the array by most restrictive cidr first + network = network.sort_by{ |net| -net.prefix() } network.compact! #clean nulls # Try every combination of address and network, first match wins address.product(network).each do |a, n| From 4aef06b899493280cc60a2e660169c681b3607ea Mon Sep 17 00:00:00 2001 From: rkbennett <44292326+rkbennett@users.noreply.github.com> Date: Tue, 2 Feb 2021 22:38:03 -0600 Subject: [PATCH 2/3] Move network array sort to after null clean Change sorts the network array by cider descending When used in combination with the first match wins method, will always result in most restrictive match being selected --- lib/logstash/filters/cidr.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/logstash/filters/cidr.rb b/lib/logstash/filters/cidr.rb index 973a72d..b9211ef 100644 --- a/lib/logstash/filters/cidr.rb +++ b/lib/logstash/filters/cidr.rb @@ -163,9 +163,9 @@ def filter(event) end end end + network.compact! #clean nulls # Sort the array by most restrictive cidr first network = network.sort_by{ |net| -net.prefix() } - network.compact! #clean nulls # Try every combination of address and network, first match wins address.product(network).each do |a, n| @logger.debug("Checking IP inclusion", :address => a, :network => n) From e738ff53cbcf43f26b8e89d008fae82e5fa9654d Mon Sep 17 00:00:00 2001 From: rkbennett <44292326+rkbennett@users.noreply.github.com> Date: Tue, 9 Feb 2021 16:47:29 -0600 Subject: [PATCH 3/3] Adds hash support Add the following configuration settings: network_type => "Array||Hash" (type for the network object, defaults to Array) network_return => true||false (do you want to return the value of a matching subnet key, defaults to false) target => "anystring" (the destination field for the returned value, defaults to 'result') You can then point the network_path to a json formatted file which would be structured something like this: { "10.1.0.0/16": "my internal network", "192.168.1.0/24": "my other internal network" } Alternatively, you can just call the plugin from within the pipeline like this: cidr { address=> [ "10.1.1.1" ] target => "network" network_return => true network => { "10.1.0.0/16" => "my internal network" } } You are also able to nest hashes inside the network hash keys, allowing for even more advanced tagging. --- lib/logstash/filters/cidr.rb | 174 ++++++++++++++++++++++++++++------- 1 file changed, 139 insertions(+), 35 deletions(-) diff --git a/lib/logstash/filters/cidr.rb b/lib/logstash/filters/cidr.rb index b9211ef..d5c0ee7 100644 --- a/lib/logstash/filters/cidr.rb +++ b/lib/logstash/filters/cidr.rb @@ -2,6 +2,7 @@ require "logstash/filters/base" require "logstash/namespace" require "ipaddr" +require "json" # The CIDR filter is for checking IP addresses in events against a list of # network blocks that might contain it. Multiple addresses can be checked @@ -33,7 +34,49 @@ class LogStash::Filters::CIDR < LogStash::Filters::Base # network => [ "169.254.0.0/16", "fe80::/64" ] # } # } - config :network, :validate => :array, :default => [] + config :network, :validate => :array || :hash , :default => [] + + # The type of data in the external file containing the IP network(s) to check against. Can be "Array" or "Hash". + # If set to "Hash", file must be json formatted. Does nothing unless network_path is set. Example: + # [source,ruby] + # filter { + # %PLUGIN% { + # add_tag => [ "linklocal" ] + # address => [ "%{clientip}" ] + # network_type => "Hash" + # network_path => "/etc/logstash/networks.json" + # } + # } + config :network_type, :validate => :string, :default => "Array" + + # Return the data corresponding to the network(s) Hash(es). + # If set to true, corresponding hash will be returned to a specified field, or "result" if none are specified. Does nothing unless network_type is Hash. Example: + # [source,ruby] + # filter { + # %PLUGIN% { + # add_tag => [ "linklocal" ] + # address => [ "%{clientip}" ] + # network_type => "Hash" + # network_path => "/etc/logstash/networks.json" + # network_return => true + # } + # } + config :network_return, :validate => :boolean, :default => false + + # The field with which to return the data corresponding to the network(s) Hash(es). + # If set, corresponding hash will be returned to a specified field, "result" if not specified. Does nothing unless network_type is Hash. Example: + # [source,ruby] + # filter { + # %PLUGIN% { + # add_tag => [ "linklocal" ] + # address => [ "%{clientip}" ] + # network_type => "Hash" + # network_path => "/etc/logstash/networks.json" + # network_return => true + # target => "FieldName" + # } + # } + config :target, :validate => :string, :default => "result" # The full path of the external file containing the IP network(s) to check against. Example: # [source,ruby] @@ -100,22 +143,50 @@ def needs_refresh? end # def needs_refresh def load_file - begin - temporary = File.open(@network_path, "r") {|file| file.read.split(@separator)} - if !temporary.empty? #ensuring the file was parsed correctly - @network_list = temporary + if @network_type.eql?("Hash") + begin + json_temporary = File.read(@network_path) + temporary = JSON.parse(json_temporary) + if !temporary.empty? #ensuring the file was parsed correctly + @network_list = temporary + end + rescue + if @network_list #if the list was parsed successfully before + @logger.error("Error while opening/parsing the file") + else + raise LogStash::ConfigurationError, I18n.t( + "logstash.agent.configuration.invalid_plugin_register", + :plugin => "filter", + :type => "cidr", + :error => "The file containing the network list is invalid, please check the separator character or permissions for the file." + ) + end end - rescue - if @network_list #if the list was parsed successfully before - @logger.error("Error while opening/parsing the file") - else - raise LogStash::ConfigurationError, I18n.t( - "logstash.agent.configuration.invalid_plugin_register", - :plugin => "filter", - :type => "cidr", - :error => "The file containing the network list is invalid, please check the separator character or permissions for the file." - ) + elsif @network_type.eql?("Array") + begin + temporary = File.open(@network_path, "r") {|file| file.read.split(@separator)} + if !temporary.empty? #ensuring the file was parsed correctly + @network_list = temporary + end + rescue + if @network_list #if the list was parsed successfully before + @logger.error("Error while opening/parsing the file") + else + raise LogStash::ConfigurationError, I18n.t( + "logstash.agent.configuration.invalid_plugin_register", + :plugin => "filter", + :type => "cidr", + :error => "The file containing the network list is invalid, please check the separator character or permissions for the file." + ) + end end + else + raise LogStash::ConfigurationError, I18n.t( + "logstash.agent.configuration.invalid_plugin_register", + :plugin => "filter", + :type => "cidr", + :error => "When defining network_type, value must be one of Array or Hash." + ) end end #def load_file @@ -140,38 +211,71 @@ def filter(event) end end #end lock end #end refresh from file - - network = @network_list.collect do |n| - begin - lock_for_read do - IPAddr.new(n) + if @network_type.eql?("Hash") + network = @network_list.keys.collect do |n| + begin + lock_for_read do + IPAddr.new(n) + end + rescue ArgumentError => e + @logger.warn("Invalid IP network, skipping", :network => n, :event => event.to_hash) + nil + end + end + + elsif @network_type.eql?("Array") + network = @network_list.collect do |n| + begin + lock_for_read do + IPAddr.new(n) + end + rescue ArgumentError => e + @logger.warn("Invalid IP network, skipping", :network => n, :event => event.to_hash) + nil end - rescue ArgumentError => e - @logger.warn("Invalid IP network, skipping", :network => n, :event => event.to_hash) - nil end end - else #networks come from array in config file - - network = @network.map {|nw| event.sprintf(nw) }.map {|nw| nw.split(",") }.flatten.collect do |n| - begin - IPAddr.new(n.strip) - rescue ArgumentError => e - @logger.warn("Invalid IP network, skipping", :network => n, :event => event.to_hash) - nil + if @network.is_a?(Hash) + network = @network.keys {|nw| event.sprintf(nw) }.map {|nw| nw.split(",") }.flatten.collect do |n| + begin + IPAddr.new(n.strip) + rescue ArgumentError => e + @logger.warn("Invalid IP network, skipping", :network => n, :event => event.to_hash) + nil + end + end + elsif @network.is_a?(Array) + network = @network.map {|nw| event.sprintf(nw) }.map {|nw| nw.split(",") }.flatten.collect do |n| + begin + IPAddr.new(n.strip) + rescue ArgumentError => e + @logger.warn("Invalid IP network, skipping", :network => n, :event => event.to_hash) + nil + end end end + network = network.sort_by{ |net| -net.prefix() } end + network.compact! #clean nulls - # Sort the array by most restrictive cidr first - network = network.sort_by{ |net| -net.prefix() } # Try every combination of address and network, first match wins address.product(network).each do |a, n| @logger.debug("Checking IP inclusion", :address => a, :network => n) if n.include?(a) - filter_matched(event) - return + if (@network.is_a?(Hash) || @network_list.is_a?(Hash)) and @network_return + if @network.is_a?(Hash) + val = @network["#{n.to_s}/#{n.prefix()}"] + elsif @network_list.is_a?(Hash) + val = @network_list["#{n.to_s}/#{n.prefix()}"] + end + event.set(@target, val) + filter_matched(event) + return + else + filter_matched(event) + return + end end end end # def filter