From 61f81bbd1c6a7ffe2c2ed2c260f3670af06b9e6b Mon Sep 17 00:00:00 2001 From: Lars <126493657+ash-darin@users.noreply.github.com> Date: Tue, 1 Apr 2025 12:38:57 +0200 Subject: [PATCH 1/8] Support other rufus scheduling options Allow jdbc input to be scheduled with different options supported by rufus scheduler. --- lib/logstash/inputs/jdbc.rb | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/lib/logstash/inputs/jdbc.rb b/lib/logstash/inputs/jdbc.rb index 7de6ace..73b1f01 100755 --- a/lib/logstash/inputs/jdbc.rb +++ b/lib/logstash/inputs/jdbc.rb @@ -180,6 +180,20 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base # exactly once. config :schedule, :validate => :string + # Interval of how soon to run statement again after completion + # for example: "1m" (execute again 1 minute after completion) + # + # There is no interval by default. If no interval is given, then the statement is run + # exactly once. + config :interval, :validate => :string + + # Start the job periodically after time elapsed + # for example: "1m" (execute query every minute) + # + # There is no period by default. If no period is given, then the statement is run + # exactly once. + config :period, :validate => :string + # Path to file with last run time. # The default will write file to `/plugins/inputs/jdbc/logstash_jdbc_last_run` # NOTE: it must be a file path and not a directory path @@ -322,6 +336,12 @@ def run(queue) # scheduler input thread name example: "[my-oracle]|input|jdbc|scheduler" scheduler.cron(@schedule) { execute_query(queue) } scheduler.join + elsif @interval + scheduler.interval(@interval) { execute_query(queue) } + scheduler.join + elsif @period + scheduler.every(@period) { execute_query(queue) } + scheduler.join else execute_query(queue) end From e9b389f94ca1603f8679d124ca1d9fc41c86be8f Mon Sep 17 00:00:00 2001 From: Lars <126493657+ash-darin@users.noreply.github.com> Date: Thu, 24 Apr 2025 12:40:03 +0200 Subject: [PATCH 2/8] Document new scheduling options --- docs/input-jdbc.asciidoc | 52 ++++++++++++++++++++++++++++++++++------ 1 file changed, 45 insertions(+), 7 deletions(-) diff --git a/docs/input-jdbc.asciidoc b/docs/input-jdbc.asciidoc index d1bdb33..c0d0a05 100644 --- a/docs/input-jdbc.asciidoc +++ b/docs/input-jdbc.asciidoc @@ -41,9 +41,9 @@ options for more info. Input from this plugin can be scheduled to run periodically according to a specific schedule. This scheduling syntax is powered by https://github.com/jmettraux/rufus-scheduler[rufus-scheduler]. -The syntax is cron-like with some extensions specific to Rufus (e.g. timezone support ). +The syntax is either cron-like with some extensions specific to Rufus (e.g. timezone support ) if using the `schedule` option or periodic when using `period` or `interval` option. -Examples: +Examples for `schedule`: |========================================================== | `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March. @@ -51,9 +51,21 @@ Examples: | `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day. |========================================================== +Examples for `period` or `interval`: + +|========================================================== +| `1m` | will execute every minute +| `3h10m` | will execute every three hours and 10 minutes +|========================================================== Further documentation describing this syntax can be found https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[here]. +`interval` jobs trigger, execute and then trigger again after the interval elapsed. + +`period` jobs try to trigger following the frequency they were scheduled with. + +If you provide more than one scheduling statement, then `schedule` takes precedence over `interval` which takes precedence over `period`. + ==== State The plugin will persist the `sql_last_value` parameter in the form of a @@ -212,6 +224,7 @@ This plugin supports the following configuration options plus the <> |<>|No | <> |<>|No | <> |<>|No +| <> |<>|No | <> |<>|Yes | <> |<>|No | <> |<>|Yes @@ -229,6 +242,7 @@ This plugin supports the following configuration options plus the <> |<>|No | <> |<>|No | <> |<>|No +| <> |<>|No | <> |<>, one of `["local", "utc"]`|No | <> |<>|No | <> |<>|No @@ -294,10 +308,22 @@ Maximum number of times to try connecting to database ===== `connection_retry_attempts_wait_time` * Value type is <> - * Default value is `0.5` + * Default value is 0.5 Number of seconds to sleep between connection attempts +[id="plugins-{type}s-{plugin}-interval"] +===== `interval` + + * Value type is <> + * There is no default value for this setting. + +This takes a string in the form of `1h`, `1m`, to denote a time interval. `interval` jobs trigger, execute and trigger again after the provided time interval has elapsed. + +There is no schedule by default. If no scheduling statement is given, then the statement is run exactly once. + +If you provide more than one scheduling statement, then <> takes precedence over `interval` which takes precedence over <>. + [id="plugins-{type}s-{plugin}-jdbc_connection_string"] ===== `jdbc_connection_string` @@ -538,6 +564,18 @@ Whether to force the lowercasing of identifier fields Hash of query parameter, for example `{ "target_id" => "321" }` +[id="plugins-{type}s-{plugin}-period"] +===== `period` + + * Value type is <> + * There is no default value for this setting. + +This takes a string in the form of `1h`, `1m`, to denote a time interval. `period` jobs try hard to trigger following the frequency they were scheduled with. + +There is no schedule by default. If no scheduling statement is given, then the statement is run exactly once. + +If you provide more than one scheduling statement, then <> takes precedence over <> which takes precedence over `period`. + [id="plugins-{type}s-{plugin}-prepared_statement_bind_values"] ===== `prepared_statement_bind_values` @@ -568,11 +606,11 @@ Whether to save state or not in <> * There is no default value for this setting. -Schedule of when to periodically run statement, in Cron format -for example: "* * * * *" (execute query every minute, on the minute) +Schedule of when to periodically run statement, in Cron format for example: "* * * * *" (execute query every minute, on the minute) + +There is no schedule by default. If no scheduling statement is given, then the statement is run exactly once. -There is no schedule by default. If no schedule is given, then the statement is run -exactly once. +If you provide more than one scheduling statement, then `schedule` takes precedence over <> which takes precedence over <>. [id="plugins-{type}s-{plugin}-sequel_opts"] ===== `sequel_opts` From c9bd423c6364cc6de144c670f8d01e63951fe539 Mon Sep 17 00:00:00 2001 From: Lars <126493657+ash-darin@users.noreply.github.com> Date: Thu, 24 Apr 2025 13:02:38 +0200 Subject: [PATCH 3/8] revert accidental formatting loss --- docs/input-jdbc.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/input-jdbc.asciidoc b/docs/input-jdbc.asciidoc index c0d0a05..fa9486f 100644 --- a/docs/input-jdbc.asciidoc +++ b/docs/input-jdbc.asciidoc @@ -308,7 +308,7 @@ Maximum number of times to try connecting to database ===== `connection_retry_attempts_wait_time` * Value type is <> - * Default value is 0.5 + * Default value is `0.5` Number of seconds to sleep between connection attempts From 65335b8099e88c384c6e0919a1d357a927b5cb91 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Mon, 28 Apr 2025 10:31:51 +0100 Subject: [PATCH 4/8] make scheduling option exclusivity explicit --- docs/input-jdbc.asciidoc | 8 +------- lib/logstash/inputs/jdbc.rb | 5 +++++ spec/inputs/jdbc_spec.rb | 26 ++++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/docs/input-jdbc.asciidoc b/docs/input-jdbc.asciidoc index fa9486f..1a19de0 100644 --- a/docs/input-jdbc.asciidoc +++ b/docs/input-jdbc.asciidoc @@ -64,7 +64,7 @@ Further documentation describing this syntax can be found https://github.com/jme `period` jobs try to trigger following the frequency they were scheduled with. -If you provide more than one scheduling statement, then `schedule` takes precedence over `interval` which takes precedence over `period`. +You can only use of `interval`, `period` or `schedule` at the same time. ==== State @@ -322,8 +322,6 @@ This takes a string in the form of `1h`, `1m`, to denote a time interval. `inter There is no schedule by default. If no scheduling statement is given, then the statement is run exactly once. -If you provide more than one scheduling statement, then <> takes precedence over `interval` which takes precedence over <>. - [id="plugins-{type}s-{plugin}-jdbc_connection_string"] ===== `jdbc_connection_string` @@ -574,8 +572,6 @@ This takes a string in the form of `1h`, `1m`, to denote a time interval. `perio There is no schedule by default. If no scheduling statement is given, then the statement is run exactly once. -If you provide more than one scheduling statement, then <> takes precedence over <> which takes precedence over `period`. - [id="plugins-{type}s-{plugin}-prepared_statement_bind_values"] ===== `prepared_statement_bind_values` @@ -610,8 +606,6 @@ Schedule of when to periodically run statement, in Cron format for example: "* * There is no schedule by default. If no scheduling statement is given, then the statement is run exactly once. -If you provide more than one scheduling statement, then `schedule` takes precedence over <> which takes precedence over <>. - [id="plugins-{type}s-{plugin}-sequel_opts"] ===== `sequel_opts` diff --git a/lib/logstash/inputs/jdbc.rb b/lib/logstash/inputs/jdbc.rb index 73b1f01..3cf93dc 100755 --- a/lib/logstash/inputs/jdbc.rb +++ b/lib/logstash/inputs/jdbc.rb @@ -255,6 +255,11 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base def register @logger = self.logger + if [@interval, @schedule, @period].compact.size > 1 + raise LogStash::ConfigurationError.new("Use only one of: interval, period, schedule.") + end + + if @record_last_run if @last_run_metadata_path.nil? logstash_data_path = LogStash::SETTINGS.get_value("path.data") diff --git a/spec/inputs/jdbc_spec.rb b/spec/inputs/jdbc_spec.rb index f4a9880..dd829da 100755 --- a/spec/inputs/jdbc_spec.rb +++ b/spec/inputs/jdbc_spec.rb @@ -228,6 +228,32 @@ end end + describe "scheduling options" do + scheduling_options = ["interval", "schedule", "period"] + scheduling_options.combination(2).each do |option1, option2| + context "when using '#{option1}' and '#{option2}' at the same time" do + let(:settings) { super().merge(option1 => 'a', option2 => 'b') } + it "raises a configuration error" do + expect { plugin.register }.to raise_error(LogStash::ConfigurationError) + end + end + end + context "when using 'schedule', 'period' and 'interval' at the same time" do + let(:settings) { super().merge("interval" => "a", "period" => "b", "schedule" => "c") } + it "raises a configuration error" do + expect { plugin.register }.to raise_error(LogStash::ConfigurationError) + end + end + scheduling_options.each do |option| + context "when using only '#{option}'" do + let(:settings) { super().merge(option => "a") } + it "does not raise a configuration error" do + expect { plugin.register }.to_not raise_error + end + end + end + end + context "when scheduling" do let(:settings) { {"statement" => "SELECT 1 as num_param FROM SYSIBM.SYSDUMMY1", "schedule" => "* * * * * UTC"} } From 3bf636048bf5bdb3ccae60546316491d83bb8063 Mon Sep 17 00:00:00 2001 From: Lars <126493657+ash-darin@users.noreply.github.com> Date: Wed, 28 May 2025 10:05:45 +0200 Subject: [PATCH 5/8] Update input-jdbc.asciidoc --- docs/input-jdbc.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/input-jdbc.asciidoc b/docs/input-jdbc.asciidoc index 1a19de0..aebdd53 100644 --- a/docs/input-jdbc.asciidoc +++ b/docs/input-jdbc.asciidoc @@ -1,4 +1,4 @@ -:integration: jdbc +⁰:integration: jdbc :plugin: jdbc :type: input :default_codec: plain @@ -64,7 +64,7 @@ Further documentation describing this syntax can be found https://github.com/jme `period` jobs try to trigger following the frequency they were scheduled with. -You can only use of `interval`, `period` or `schedule` at the same time. +You can only use one of `interval`, `period` or `schedule` at the same time. ==== State From a17b0dadd8ab49ecb17defe2aa86f25891e3d6fb Mon Sep 17 00:00:00 2001 From: Lars <126493657+ash-darin@users.noreply.github.com> Date: Wed, 28 May 2025 10:10:03 +0200 Subject: [PATCH 6/8] Update input-jdbc.asciidoc --- docs/input-jdbc.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/input-jdbc.asciidoc b/docs/input-jdbc.asciidoc index aebdd53..dee0f84 100644 --- a/docs/input-jdbc.asciidoc +++ b/docs/input-jdbc.asciidoc @@ -1,4 +1,4 @@ -⁰:integration: jdbc +:integration: jdbc :plugin: jdbc :type: input :default_codec: plain From 9f124825953d27b325ca5dc8165d06c1bac35bf2 Mon Sep 17 00:00:00 2001 From: Lars <126493657+ash-darin@users.noreply.github.com> Date: Wed, 28 May 2025 11:54:27 +0200 Subject: [PATCH 7/8] Update jdbc_spec.rb Remove default "no error case", build fails otherwise --- spec/inputs/jdbc_spec.rb | 8 -------- 1 file changed, 8 deletions(-) diff --git a/spec/inputs/jdbc_spec.rb b/spec/inputs/jdbc_spec.rb index dd829da..6085c41 100755 --- a/spec/inputs/jdbc_spec.rb +++ b/spec/inputs/jdbc_spec.rb @@ -244,14 +244,6 @@ expect { plugin.register }.to raise_error(LogStash::ConfigurationError) end end - scheduling_options.each do |option| - context "when using only '#{option}'" do - let(:settings) { super().merge(option => "a") } - it "does not raise a configuration error" do - expect { plugin.register }.to_not raise_error - end - end - end end context "when scheduling" do From 9c853a5976642c97de371ab192928ddc01251e33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Duarte?= Date: Fri, 30 May 2025 10:45:35 +0100 Subject: [PATCH 8/8] Update jdbc_spec.rb --- spec/inputs/jdbc_spec.rb | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/spec/inputs/jdbc_spec.rb b/spec/inputs/jdbc_spec.rb index 6085c41..4d08ccd 100755 --- a/spec/inputs/jdbc_spec.rb +++ b/spec/inputs/jdbc_spec.rb @@ -229,19 +229,28 @@ end describe "scheduling options" do + let(:settings) { super().merge("statement" => "SELECT :num_param as num_param FROM SYSIBM.SYSDUMMY1") } scheduling_options = ["interval", "schedule", "period"] scheduling_options.combination(2).each do |option1, option2| context "when using '#{option1}' and '#{option2}' at the same time" do let(:settings) { super().merge(option1 => 'a', option2 => 'b') } it "raises a configuration error" do - expect { plugin.register }.to raise_error(LogStash::ConfigurationError) + expect { plugin.register }.to raise_error(LogStash::ConfigurationError, /Use only one/) end end end context "when using 'schedule', 'period' and 'interval' at the same time" do let(:settings) { super().merge("interval" => "a", "period" => "b", "schedule" => "c") } it "raises a configuration error" do - expect { plugin.register }.to raise_error(LogStash::ConfigurationError) + expect { plugin.register }.to raise_error(LogStash::ConfigurationError, /Use only one/) + end + end + scheduling_options.each do |option| + context "when using only '#{option}'" do + let(:settings) { super().merge(option => "a") } + it "does not raise a configuration error" do + expect { plugin.register }.to_not raise_error + end end end end