Skip to content

Commit ab44003

Browse files
authored
Add input plugin (#7)
We can specify any tag for the filter because filter plugin does nothing. However it would cause unnecessary method calls for `filter`. It is preferable to run as an input plugin for simpler.
1 parent 608b968 commit ab44003

File tree

6 files changed

+176
-10
lines changed

6 files changed

+176
-10
lines changed

README.md

Lines changed: 27 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -30,27 +30,49 @@ $ bundle
3030

3131
## Configuration
3232

33-
### plugins_json (string) (optional)
33+
### Filter plugin
34+
35+
#### plugins_json (string) (optional)
3436

3537
Path to `plugins.json`.
3638

3739
Default value: `https://raw.githubusercontent.com/fluent/fluentd-website/master/scripts/plugins.json`.
3840

39-
40-
### Deprecated: obsolete_plugins_yml (string) (optional)
41+
#### Deprecated: obsolete_plugins_yml (string) (optional)
4142

4243
Path to `obsolete-plugins.yml`. This parameter is deprecated. Please use `plugins_json` parameter instead.
4344

4445
Default value: `nil`
4546

46-
### timeout (integer) (optional)
47+
#### timeout (integer) (optional)
48+
49+
Timeout to read data of obsolete plugins.
50+
If it occurs timeout, it just skips to detect obsolete plugins.
51+
52+
Default value: `5`
53+
54+
#### raise_error (bool) (optional)
55+
56+
Raise error if obsolete plugins are detected
57+
58+
Default value: `false`.
59+
60+
### Input plugin
61+
62+
#### plugins_json (string) (optional)
63+
64+
Path to `plugins.json`.
65+
66+
Default value: `https://raw.githubusercontent.com/fluent/fluentd-website/master/scripts/plugins.json`.
67+
68+
#### timeout (integer) (optional)
4769

4870
Timeout to read data of obsolete plugins.
4971
If it occurs timeout, it just skips to detect obsolete plugins.
5072

5173
Default value: `5`
5274

53-
### raise_error (bool) (optional)
75+
#### raise_error (bool) (optional)
5476

5577
Raise error if obsolete plugins are detected
5678

lib/fluent/plugin/filter_obsolete_plugins.rb

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ def configure(conf)
4646
rescue Fluent::ConfigError
4747
raise
4848
rescue => e
49-
log.info("Failed to notfify obsolete plugins", error: e)
49+
log.info("Failed to notify obsolete plugins", error: e)
5050
end
5151

5252
def filter(tag, time, record)
Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
require "fluent/plugin/input"
2+
require "fluent/plugin/obsolete_plugins_utils"
3+
4+
module Fluent
5+
module Plugin
6+
class ObsoletePluginsInput < Fluent::Plugin::Input
7+
Fluent::Plugin.register_input("obsolete_plugins", self)
8+
9+
PLUGINS_JSON_URL = "https://raw.githubusercontent.com/fluent/fluentd-website/master/scripts/plugins.json"
10+
11+
desc "Path to plugins.json"
12+
config_param :plugins_json, :string, default: PLUGINS_JSON_URL
13+
desc "Timeout value to read data of obsolete plugins"
14+
config_param :timeout, :integer, default: 5
15+
desc "Raise error if obsolete plugins are detected"
16+
config_param :raise_error, :bool, default: false
17+
18+
def configure(conf)
19+
super
20+
21+
obsolete_plugins = ObsoletePluginsUtils.obsolete_plugins_from_json(@plugins_json, timeout: @timeout)
22+
ObsoletePluginsUtils.notify(log, obsolete_plugins, raise_error: @raise_error)
23+
rescue Fluent::ConfigError
24+
raise
25+
rescue => e
26+
log.info("Failed to notify obsolete plugins", error: e)
27+
end
28+
end
29+
end
30+
end

test/helper.rb

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
require "test/unit/rr"
44
require "timecop"
55
require "fluent/test"
6-
require "fluent/test/driver/filter"
76
require "fluent/test/helpers"
87

98
Test::Unit::TestCase.include(Fluent::Test::Helpers)

test/plugin/test_filter_obsolete_plugins.rb

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
require "helper"
2+
require "fluent/test/driver/filter"
23
require "fluent/plugin/filter_obsolete_plugins"
34

45
class ObsoletePluginsFilterTest < Test::Unit::TestCase
56

67
setup do
78
Fluent::Test.setup
8-
$log = Fluent::Test::TestLogger.new
99
@time = Time.now
1010
Timecop.freeze(@time)
1111
end
@@ -119,7 +119,7 @@ class ObsoletePluginsFilterTest < Test::Unit::TestCase
119119
d = create_driver("plugins_json #{fixture_path('invalid.json')}")
120120

121121
expected_logs = [
122-
"#{@time} [info]: Failed to notfify obsolete plugins error_class=JSON::ParserError error=\"expected ',' or '}' after object value, got: EOF at line 11 column 1\"\n",
122+
"#{@time} [info]: Failed to notify obsolete plugins error_class=JSON::ParserError error=\"expected ',' or '}' after object value, got: EOF at line 11 column 1\"\n",
123123
]
124124

125125
assert_equal(expected_logs, d.logs)
@@ -138,7 +138,7 @@ class ObsoletePluginsFilterTest < Test::Unit::TestCase
138138
sleep 2
139139

140140
expected_logs = [
141-
"#{@time} [info]: Failed to notfify obsolete plugins error_class=Timeout::Error error=\"execution expired\"\n",
141+
"#{@time} [info]: Failed to notify obsolete plugins error_class=Timeout::Error error=\"execution expired\"\n",
142142
]
143143

144144
assert_equal(expected_logs, d.logs)
Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
require "helper"
2+
require "fluent/test/driver/input"
3+
require "fluent/plugin/in_obsolete_plugins"
4+
require "fluent/plugin/obsolete_plugins_utils"
5+
6+
class ObsoletePluginsInputTest < Test::Unit::TestCase
7+
8+
setup do
9+
Fluent::Test.setup
10+
@time = Time.now
11+
Timecop.freeze(@time)
12+
end
13+
14+
teardown do
15+
Timecop.return
16+
end
17+
18+
sub_test_case "plugins_json" do
19+
CONFIG_JSON = %[
20+
plugins_json #{fixture_path("plugins.json")}
21+
]
22+
23+
test "no obsolete plugins" do
24+
d = create_driver(CONFIG_JSON)
25+
d.run
26+
assert_equal([], d.events)
27+
assert_equal([], d.logs)
28+
end
29+
30+
test "obsolete plugins" do
31+
stub(Gem).loaded_specs do
32+
{
33+
"fluent-plugin-tail-multiline" => nil,
34+
"fluent-plugin-hostname" => nil
35+
}
36+
end
37+
d = create_driver(CONFIG_JSON)
38+
d.run
39+
assert_equal([], d.events)
40+
expected_logs = [
41+
"#{@time} [warn]: fluent-plugin-tail-multiline is obsolete: Merged in in_tail in Fluentd v0.10.45. [fluent/fluentd#269](https://github.com/fluent/fluentd/issues/269)\n",
42+
"#{@time} [warn]: fluent-plugin-hostname is obsolete: Use [filter\\_record\\_transformer](http://docs.fluentd.org/v0.12/articles/filter_record_transformer) instead.\n"
43+
]
44+
assert_equal(expected_logs, d.logs)
45+
end
46+
47+
test "raise error when detect obsolete plugins" do
48+
stub(Gem).loaded_specs do
49+
{
50+
"fluent-plugin-tail-multiline" => nil,
51+
"fluent-plugin-hostname" => nil
52+
}
53+
end
54+
55+
ex = assert_raise(Fluent::ConfigError) do
56+
create_driver(CONFIG_JSON + "raise_error yes")
57+
end
58+
assert_equal("Detected obsolete plugins", ex.message)
59+
end
60+
end
61+
62+
sub_test_case "error handling" do
63+
test "invalid json" do
64+
d = create_driver("plugins_json #{fixture_path('invalid.json')}")
65+
66+
expected_logs = [
67+
"#{@time} [info]: Failed to notify obsolete plugins error_class=JSON::ParserError error=\"expected ',' or '}' after object value, got: EOF at line 11 column 1\"\n",
68+
]
69+
70+
assert_equal(expected_logs, d.logs)
71+
end
72+
73+
test "timeout with slow server" do
74+
server = create_slow_webserver(port: 12345)
75+
76+
mock(Fluent::Plugin::ObsoletePluginsUtils).notify.never
77+
78+
d = create_driver(%[
79+
plugins_json http://localhost:12345/plugins.json
80+
timeout 1
81+
])
82+
83+
sleep 2
84+
85+
expected_logs = [
86+
"#{@time} [info]: Failed to notify obsolete plugins error_class=Timeout::Error error=\"execution expired\"\n",
87+
]
88+
89+
assert_equal(expected_logs, d.logs)
90+
ensure
91+
server.shutdown
92+
end
93+
94+
end
95+
96+
private
97+
98+
def create_driver(conf)
99+
Fluent::Test::Driver::Input.new(Fluent::Plugin::ObsoletePluginsInput).configure(conf)
100+
end
101+
102+
def create_slow_webserver(port: 12345)
103+
require "webrick"
104+
105+
server = WEBrick::HTTPServer.new(Port: port)
106+
server.mount_proc '/' do |req, res|
107+
sleep 60
108+
109+
res['Content-Type'] = 'application/json'
110+
res.body = File.read(fixture_path("plugins.json"))
111+
end
112+
113+
server
114+
end
115+
end

0 commit comments

Comments
 (0)