Skip to content

Commit dfecd55

Browse files
author
Jarret Lavallee
committed
(SUP-1969) Enable metrics shipping for system metrics
Prior to this commit, there was no way to ship the system level metrics to a remote metrics server. This commit adds that capability to keep it inline with the pe_metrics capabilities.
1 parent 2477df7 commit dfecd55

File tree

9 files changed

+176
-11
lines changed

9 files changed

+176
-11
lines changed

‎files/system_metrics‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -54,13 +54,14 @@ module SystemMetrics
5454
#
5555
# @return [void]
5656
def initialize(polling_interval, file_interval, metric_type, process_expression, metrics_dir,
57-
verbose = false)
57+
verbose = false, print_output = false)
5858
@polling_interval = polling_interval
5959
@file_interval = file_interval
6060
@metric_type = metric_type
6161
@process_expression = process_expression
6262
@metrics_dir = metrics_dir
6363
@verbose = verbose
64+
@print_output = print_output
6465

6566
@hostname = `hostname`.strip
6667
puts "Hostname is: #{@hostname}" if @verbose
@@ -398,6 +399,9 @@ module SystemMetrics
398399

399400
metrics_json = metrics_to_json(metrics_data)
400401

402+
if @print_output != false then
403+
STDOUT.puts(metrics_json)
404+
end
401405
write_metrics_to_file(metrics_json)
402406
end
403407
end
@@ -427,12 +431,13 @@ if $PROGRAM_NAME == __FILE__
427431
* process_expression (-e, --process_expression): #{PROCESS_EXPRESSION_DEFAULT}
428432
* metrics_dir (-m, --metrics_dir): #{METRICS_DIR_DEFAULT}
429433
* verbose (-v, --verbose): False
434+
* Print to STDOUT (--[no-]print): False
430435
DEFAULTS
431436

432437
options = {}
433438

434439
OptionParser.new { |opts|
435-
opts.banner = 'Usage: generate_system_stats.rb [options]'
440+
opts.banner = "Usage: system_metrics [options]"
436441

437442
opts.on('-h', '--help', 'Display the help text') do
438443
puts DESCRIPTION
@@ -461,7 +466,8 @@ if $PROGRAM_NAME == __FILE__
461466
'The puppet_metrics_collector output directory') do |metrics_dir|
462467
options[:metrics_dir] = metrics_dir
463468
end
464-
opts.on('-v', '--verbose', String, 'Enable Verbose output') { options[:verbose] = true }
469+
opts.on("-v", "--verbose", String, "Enable Verbose output") { options[:verbose] = true }
470+
opts.on('--[no-]print', 'Print to STDOUT') { |p| options[:print] = p }
465471
}.parse!
466472

467473
if options[:metric_type]
@@ -478,6 +484,7 @@ if $PROGRAM_NAME == __FILE__
478484
process_expression = options[:process_expression] || PROCESS_EXPRESSION_DEFAULT
479485
metrics_dir = options[:metrics_dir] || METRICS_DIR_DEFAULT
480486
verbose = options[:verbose] || false
487+
print_output = options[:print] || false
481488

482489
if options[:polling_interval] || options[:file_interval]
483490
options_error = 'Polling interval must be less than file interval'
@@ -493,11 +500,12 @@ if $PROGRAM_NAME == __FILE__
493500
* process_expression #{process_expression}
494501
* metrics_dir: #{metrics_dir}
495502
* verbose: #{verbose}
503+
* print: #{print_output}
496504
SETTINGS
497505
puts OPTION_SETTINGS
498506
end
499507

500508
obj = SystemMetrics::GenerateSystemMetrics.new(polling_interval, file_interval, metric_type,
501-
process_expression, metrics_dir, verbose)
509+
process_expression, metrics_dir, verbose, print_output)
502510
obj.generate_metrics
503511
end

‎functions/generate_metrics_server_command.pp‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
# @summary Generate the metrics shipping command for the cron job including remote metrics
2+
# @param scripts_dir the path to the scripts directory
3+
# @param metrics_server_type the metric server type
4+
# @param metrics_server_hostname the metric server's address
5+
# @param metrics_server_db_name the influxdb database name
6+
# @param metrics_server_port the port to connect to
7+
# @return [String] of the metrics command or undef
8+
function puppet_metrics_collector::generate_metrics_server_command (
9+
Optional[String] $scripts_dir,
10+
Optional[Enum['influxdb','graphite','splunk_hec']] $metrics_server_type = undef,
11+
Optional[String] $metrics_server_hostname = undef,
12+
Optional[String] $metrics_server_db_name = undef,
13+
Optional[Integer] $metrics_server_port = undef,
14+
) >> String {
15+
if !empty($metrics_server_type) {
16+
if empty($metrics_server_db_name) and $metrics_server_type == 'influxdb' {
17+
fail('When specifying an InfluxDB metrics server, you must specify a metrics server db_name')
18+
}
19+
20+
$port_command = empty($metrics_server_port) ? {
21+
false => "--port ${metrics_server_port}",
22+
true => undef,
23+
}
24+
25+
# We use only the base metrics command for the 'splunk_hec' metrics server type.
26+
27+
$metrics_shipping_command = $metrics_server_type ? {
28+
'influxdb' => ['--print |',
29+
"${scripts_dir}/json2timeseriesdb",
30+
"--netcat ${metrics_server_hostname}",
31+
"--convert-to ${metrics_server_type}",
32+
"--influx-db ${metrics_server_db_name}",
33+
$port_command,
34+
'-',
35+
].filter |$v| { $v != undef }.join(' '), # Filter out undef without stdlib
36+
'graphite' => join(['--print |',
37+
"${scripts_dir}/json2timeseriesdb",
38+
"--netcat ${metrics_server_hostname}",
39+
"--convert-to ${metrics_server_type}",
40+
'-',
41+
], ' '),
42+
'splunk_hec' => join(['--print |',
43+
'/opt/puppetlabs/bin/puppet',
44+
'splunk_hec',
45+
'--sourcetype puppet:metrics',
46+
'--pe_metrics',
47+
], ' '),
48+
default => '',
49+
}
50+
} else {
51+
$metrics_shipping_command = ''
52+
}
53+
}

‎manifests/sar_metric.pp‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
Integer $collection_frequency = 5, # minutes
88
Integer $polling_frequency_seconds = 1,
99
String $metric_script_file = 'system_metrics',
10+
String $metrics_shipping_command = $puppet_metrics_collector::system::metrics_shipping_command,
1011
) {
1112

1213
$metrics_output_dir = "${puppet_metrics_collector::system::output_dir}/${metrics_type}"
@@ -25,13 +26,14 @@
2526
$metric_script_file_path = "${puppet_metrics_collector::system::scripts_dir}/${metric_script_file}"
2627
$file_interval_seconds = $collection_frequency * 60
2728

28-
$metrics_command = join([$metric_script_file_path,
29-
" --metric_type ${metrics_type}",
30-
" --file_interval ${file_interval_seconds}",
31-
" --polling_interval ${polling_frequency_seconds}",
32-
" --metrics_dir ${puppet_metrics_collector::system::output_dir}",
33-
' > /dev/null',
34-
], '')
29+
$base_metrics_command = join([$metric_script_file_path,
30+
"--metric_type ${metrics_type}",
31+
"--file_interval ${file_interval_seconds}",
32+
"--polling_interval ${polling_frequency_seconds}",
33+
"--metrics_dir ${puppet_metrics_collector::system::output_dir}",
34+
], ' ')
35+
36+
$metrics_command = "${base_metrics_command} ${metrics_shipping_command} > /dev/null"
3537

3638
# The hardcoded numbers with the fqdn_rand calls are to trigger the metrics_tidy
3739
# command to run at a randomly selected time between 12:00 AM and 3:00 AM.

‎manifests/system.pp‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,10 @@
88
Boolean $manage_sysstat = false,
99
Boolean $manage_vmware_tools = false,
1010
String $vmware_tools_pkg = 'open-vm-tools',
11+
Optional[Enum['influxdb', 'graphite', 'splunk_hec']] $metrics_server_type = getvar('puppet_metrics_collector::metrics_server_type'),
12+
Optional[String] $metrics_server_hostname = getvar('puppet_metrics_collector::metrics_server_hostname'),
13+
Optional[Integer] $metrics_server_port = getvar('puppet_metrics_collector::metrics_server_port'),
14+
Optional[String] $metrics_server_db_name = getvar('puppet_metrics_collector::metrics_server_db_name'),
1115
) {
1216
$scripts_dir = "${output_dir}/scripts"
1317

@@ -38,6 +42,14 @@
3842
refreshonly => true,
3943
}
4044

45+
$metrics_shipping_command = puppet_metrics_collector::generate_metrics_server_command(
46+
$scripts_dir,
47+
$metrics_server_type,
48+
$metrics_server_hostname,
49+
$metrics_server_db_name,
50+
$metrics_server_port
51+
)
52+
4153
if $manage_sysstat {
4254
package { 'sysstat':
4355
ensure => $system_metrics_ensure,

‎manifests/system/cpu.pp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,14 @@
44
Integer $collection_frequency = $puppet_metrics_collector::system::collection_frequency,
55
Integer $retention_days = $puppet_metrics_collector::system::retention_days,
66
Integer $polling_frequency_seconds = $puppet_metrics_collector::system::polling_frequency_seconds,
7+
Optional[String] $metrics_shipping_command = undef,
78
) {
89
puppet_metrics_collector::sar_metric { 'system_cpu' :
910
metric_ensure => $metrics_ensure,
1011
cron_minute => "0/${collection_frequency}",
1112
retention_days => $retention_days,
1213
collection_frequency => $collection_frequency,
1314
polling_frequency_seconds => $polling_frequency_seconds,
15+
metrics_shipping_command => $metrics_shipping_command,
1416
}
1517
}

‎manifests/system/memory.pp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,14 @@
44
Integer $collection_frequency = $puppet_metrics_collector::system::collection_frequency,
55
Integer $retention_days = $puppet_metrics_collector::system::retention_days,
66
Integer $polling_frequency_seconds = $puppet_metrics_collector::system::polling_frequency_seconds,
7+
Optional[String] $metrics_shipping_command = undef,
78
) {
89
puppet_metrics_collector::sar_metric { 'system_memory' :
910
metric_ensure => $metrics_ensure,
1011
cron_minute => "0/${collection_frequency}",
1112
retention_days => $retention_days,
1213
collection_frequency => $collection_frequency,
1314
polling_frequency_seconds => $polling_frequency_seconds,
15+
metrics_shipping_command => $metrics_shipping_command,
1416
}
1517
}

‎manifests/system/processes.pp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,14 @@
44
Integer $collection_frequency = $puppet_metrics_collector::system::collection_frequency,
55
Integer $retention_days = $puppet_metrics_collector::system::retention_days,
66
Integer $polling_frequency_seconds = $puppet_metrics_collector::system::polling_frequency_seconds,
7+
Optional[String] $metrics_shipping_command = undef,
78
) {
89
puppet_metrics_collector::sar_metric { 'system_processes' :
910
metric_ensure => $metrics_ensure,
1011
cron_minute => "0/${collection_frequency}",
1112
retention_days => $retention_days,
1213
collection_frequency => $collection_frequency,
1314
polling_frequency_seconds => $polling_frequency_seconds,
15+
metrics_shipping_command => $metrics_shipping_command,
1416
}
1517
}

‎spec/classes/puppet_metrics_collector_system_spec.rb‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,4 +78,39 @@
7878

7979
it { is_expected.not_to contain_service('puppet_postgres-metrics.timer') }
8080
end
81+
82+
context 'when metrics shipping is enabled' do
83+
let(:params) { {
84+
metrics_server_type: "influxdb",
85+
metrics_server_db_name: "puppet_metrics",
86+
metrics_server_hostname: "influxdb.example"
87+
}
88+
}
89+
90+
it { is_expected.to contain_cron('system_cpu_metrics_collection').with_command(/--influx-db\s+puppet_metrics/) }
91+
end
92+
93+
context 'when metrics shipping is enabled in puppet_metrics_collector' do
94+
let(:pre_condition) do
95+
<<-PRE_COND
96+
class {'puppet_metrics_collector':
97+
metrics_server_type => "influxdb",
98+
metrics_server_db_name => "puppet_metrics",
99+
metrics_server_hostname => "influxdb.example"
100+
}
101+
PRE_COND
102+
end
103+
104+
it { is_expected.to contain_cron('system_cpu_metrics_collection').with_command(/--influx-db\s+puppet_metrics/) }
105+
end
106+
107+
context 'when metrics shipping is not enabled' do
108+
let(:params) { {
109+
metrics_server_db_name: "puppet_metrics",
110+
metrics_server_hostname: "influxdb.example"
111+
}
112+
}
113+
114+
it { is_expected.not_to contain_cron('system_cpu_metrics_collection').with_command(/--influx-db\s+puppet_metrics/) }
115+
end
81116
end

‎spec/functions/generate_metrics_server_command_spec.rb‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
require 'spec_helper'
2+
3+
describe 'puppet_metrics_collector::generate_metrics_server_command' do
4+
context "when the metrics parameters are undef" do
5+
it { is_expected.to run.with_params('scripts_dir').and_return('') }
6+
end
7+
8+
context "when the metrics type is influxdb" do
9+
10+
context "without the port" do
11+
it {
12+
is_expected.to run.with_params(
13+
'scripts_dir',
14+
'influxdb',
15+
'metrics_server_hostname',
16+
'metrics_server_db_name'
17+
).and_return(
18+
"--print | scripts_dir/json2timeseriesdb --netcat metrics_server_hostname --convert-to influxdb --influx-db metrics_server_db_name -"
19+
)
20+
}
21+
end
22+
23+
context "with the port" do
24+
it {
25+
is_expected.to run.with_params(
26+
'scripts_dir',
27+
'influxdb',
28+
'metrics_server_hostname',
29+
'metrics_server_db_name',
30+
8080
31+
).and_return(
32+
"--print | scripts_dir/json2timeseriesdb --netcat metrics_server_hostname --convert-to influxdb --influx-db metrics_server_db_name --port 8080 -"
33+
)
34+
}
35+
end
36+
37+
context "without the database name" do
38+
it { is_expected.to run.with_params('scripts_dir', 'influxdb').and_raise_error(Puppet::ParseError) }
39+
end
40+
end
41+
42+
context "when metrics type is graphite" do
43+
it { is_expected.to run.with_params('scripts_dir', 'graphite', 'metrics_server_hostname').and_return('--print | scripts_dir/json2timeseriesdb --netcat metrics_server_hostname --convert-to graphite -') }
44+
end
45+
46+
context "when metrics type is splunk_hec" do
47+
it { is_expected.to run.with_params('scripts_dir', 'splunk_hec').and_return('--print | /opt/puppetlabs/bin/puppet splunk_hec --sourcetype puppet:metrics --pe_metrics') }
48+
end
49+
end

0 commit comments

Comments
 (0)