Merge pull request #94 from jarretlavallee/SUP_1969_system_metrics_shipping

jarretlavallee · web-flow · commit 84d112cc5db8 · 2021-07-01T12:10:36.000-06:00
(SUP-1969) Enable metrics shipping for all system types
diff --git a/files/psql_metrics b/files/psql_metrics
@@ -513,7 +513,8 @@ EOS
                     'Defaults to 10.'],
                    ['--output_dir DIR',
                     'Write metrics to a timestamped file under DIR instead of',
-                    'printing to STDOUT']].freeze
+                    'printing to STDOUT'],
+                   ['-p', '--[no-]print', 'Always print to STDOUT']].freeze
 
       def initialize(argv = [])
         @action = :collect_data
@@ -573,7 +574,9 @@ EOS
           FileUtils.mkdir_p(host_dir) unless File.directory?(host_dir)
           output_file = File.join(host_dir, timestamp.strftime('%Y%m%dT%H%M%SZ') + '.json')
 
-          File.write(output_file, JSON.generate(data))
+          output = JSON.generate(data)
+          File.write(output_file, output)
+          $stdout.puts(output) if @options[:print]
         else
           $stdout.puts(JSON.generate(data))
         end
diff --git a/files/system_metrics b/files/system_metrics
@@ -54,13 +54,14 @@ module SystemMetrics
     #
     # @return [void]
     def initialize(polling_interval, file_interval, metric_type, process_expression, metrics_dir,
-                   verbose = false)
+                   verbose = false, print_output = false)
       @polling_interval = polling_interval
       @file_interval = file_interval
       @metric_type = metric_type
       @process_expression = process_expression
       @metrics_dir = metrics_dir
       @verbose = verbose
+      @print_output = print_output
 
       @hostname = `hostname`.strip
       puts "Hostname is: #{@hostname}" if @verbose
@@ -398,6 +399,7 @@ module SystemMetrics
 
       metrics_json = metrics_to_json(metrics_data)
 
+      STDOUT.puts(metrics_json) if @print_output
       write_metrics_to_file(metrics_json)
     end
   end
@@ -427,12 +429,13 @@ if $PROGRAM_NAME == __FILE__
       * process_expression (-e, --process_expression): #{PROCESS_EXPRESSION_DEFAULT}
       * metrics_dir (-m, --metrics_dir): #{METRICS_DIR_DEFAULT}
       * verbose (-v, --verbose): False
+      * Print to STDOUT (--[no-]print): False
   DEFAULTS
 
   options = {}
 
   OptionParser.new { |opts|
-    opts.banner = 'Usage: generate_system_stats.rb [options]'
+    opts.banner = 'Usage: system_metrics [options]'
 
     opts.on('-h', '--help', 'Display the help text') do
       puts DESCRIPTION
@@ -462,6 +465,7 @@ if $PROGRAM_NAME == __FILE__
       options[:metrics_dir] = metrics_dir
     end
     opts.on('-v', '--verbose', String, 'Enable Verbose output') { options[:verbose] = true }
+    opts.on('--[no-]print', 'Print to STDOUT') { |p| options[:print] = p }
   }.parse!
 
   if options[:metric_type]
@@ -478,6 +482,7 @@ if $PROGRAM_NAME == __FILE__
   process_expression = options[:process_expression] || PROCESS_EXPRESSION_DEFAULT
   metrics_dir = options[:metrics_dir] || METRICS_DIR_DEFAULT
   verbose = options[:verbose] || false
+  print_output = options[:print] || false
 
   if options[:polling_interval] || options[:file_interval]
     options_error = 'Polling interval must be less than file interval'
@@ -493,11 +498,12 @@ if $PROGRAM_NAME == __FILE__
         * process_expression #{process_expression}
         * metrics_dir: #{metrics_dir}
         * verbose: #{verbose}
+        * print: #{print_output}
     SETTINGS
     puts OPTION_SETTINGS
   end
 
   obj = SystemMetrics::GenerateSystemMetrics.new(polling_interval, file_interval, metric_type,
-                                                 process_expression, metrics_dir, verbose)
+                                                 process_expression, metrics_dir, verbose, print_output)
   obj.generate_metrics
 end
diff --git a/files/vmware_metrics b/files/vmware_metrics
@@ -159,7 +159,8 @@ module PuppetMetricsCollector
                     'Defaults to 10.'],
                    ['--output_dir DIR',
                     'Write metrics to a timestamed file under DIR instead of',
-                    'printing to STDOUT']].freeze
+                    'printing to STDOUT'],
+                   ['-p', '--[no-]print', 'Always print to STDOUT']].freeze
 
       def initialize(argv = [])
         @action = :collect_data
@@ -172,7 +173,7 @@ module PuppetMetricsCollector
             @action = :show_help
           end
 
-          parser.on_tail('--debug', 'Enable backtraces from errors.') do
+          parser.on_tail('-d', '--debug', 'Enable backtraces from errors.') do
             @options[:debug] = true
           end
         end
@@ -208,7 +209,9 @@ module PuppetMetricsCollector
           FileUtils.mkdir_p(host_dir) unless File.directory?(host_dir)
           output_file = File.join(host_dir, Time.now.utc.strftime('%Y%m%dT%H%M%SZ') + '.json')
 
-          File.write(output_file, JSON.generate(data))
+          output = JSON.generate(data)
+          File.write(output_file, output)
+          $stdout.puts(output) if @options[:print]
         else
           $stdout.puts(JSON.generate(data))
         end
diff --git a/functions/generate_metrics_server_command.pp b/functions/generate_metrics_server_command.pp
@@ -0,0 +1,53 @@
+# @summary Generate the metrics shipping command for the cron job including remote metrics
+# @param scripts_dir the path to the scripts directory
+# @param metrics_server_type the metric server type
+# @param metrics_server_hostname the metric server's address
+# @param metrics_server_db_name the influxdb database name
+# @param metrics_server_port the port to connect to
+# @return [String] of the metrics command or undef
+function puppet_metrics_collector::generate_metrics_server_command (
+  Optional[String]                                   $scripts_dir,
+  Optional[Enum['influxdb','graphite','splunk_hec']] $metrics_server_type = undef,
+  Optional[String]                                   $metrics_server_hostname = undef,
+  Optional[String]                                   $metrics_server_db_name = undef,
+  Optional[Integer]                                  $metrics_server_port = undef,
+  )  >> String  {
+  if !empty($metrics_server_type) {
+    if empty($metrics_server_db_name) and $metrics_server_type == 'influxdb'  {
+      fail('When specifying an InfluxDB metrics server, you must specify a metrics server db_name')
+    }
+
+    $port_command = empty($metrics_server_port) ? {
+      false => "--port ${metrics_server_port}",
+      true  => undef,
+    }
+
+    # We use only the base metrics command for the 'splunk_hec' metrics server type.
+
+    $metrics_shipping_command = $metrics_server_type ? {
+      'influxdb'   => ['--print |',
+                      "${scripts_dir}/json2timeseriesdb",
+                      "--netcat ${metrics_server_hostname}",
+                      "--convert-to ${metrics_server_type}",
+                      "--influx-db ${metrics_server_db_name}",
+                      $port_command,
+                      '-',
+                      ].filter |$v| { $v != undef }.join(' '), # Filter out undef without stdlib
+      'graphite'   => join(['--print |',
+                      "${scripts_dir}/json2timeseriesdb",
+                      "--netcat ${metrics_server_hostname}",
+                      "--convert-to ${metrics_server_type}",
+                      '-',
+                      ], ' '),
+      'splunk_hec' => join(['--print |',
+                      '/opt/puppetlabs/bin/puppet',
+                      'splunk_hec',
+                      '--sourcetype puppet:metrics',
+                      '--pe_metrics',
+                      ], ' '),
+      default      => '',
+    }
+  } else {
+    $metrics_shipping_command = ''
+  }
+}
diff --git a/manifests/pe_metric.pp b/manifests/pe_metric.pp
@@ -62,44 +62,25 @@
   }
 
   $metric_script_file_path = "${puppet_metrics_collector::scripts_dir}/${metric_script_file}"
-  $conversion_script_file_path = "${puppet_metrics_collector::scripts_dir}/json2timeseriesdb"
 
-  # lint:ignore:140chars
   if empty($override_metrics_command) {
     $base_metrics_command = "${metric_script_file_path} --metrics_type ${metrics_type} --output_dir ${metrics_output_dir}"
+    $metrics_shipping_command = puppet_metrics_collector::generate_metrics_server_command(
+                                $puppet_metrics_collector::scripts_dir,
+                                $metrics_server_type,
+                                $metrics_server_hostname,
+                                $metrics_server_db_name,
+                                $metrics_server_port
+                              )
 
     if !empty($metrics_server_type) {
-      $server_hostname = $metrics_server_hostname
-      $server_port     = $metrics_server_port
-      $server_type     = $metrics_server_type
-      $server_db       = $metrics_server_db_name
-
-      if empty($server_db) and $server_type == 'influxdb'  {
-        fail('When specifying an InfluxDB metrics server, you must specify a metrics server db_name')
-      }
-
-      $conv_metrics_command = "${base_metrics_command} | ${conversion_script_file_path} --netcat ${server_hostname} --convert-to ${server_type} -"
-
-      $full_metrics_command = empty($server_port) ? {
-        false => "${conv_metrics_command} --port ${server_port}",
-        true  => $conv_metrics_command,
-      }
-
-      # We use only the base metrics command for the 'splunk_hec' metrics server type.
-
-      $metrics_command = $server_type ? {
-        'influxdb'   => "${full_metrics_command} --influx-db ${server_db} > /dev/null",
-        'graphite'   => "${full_metrics_command} > /dev/null",
-        'splunk_hec' => "${base_metrics_command} | /opt/puppetlabs/bin/puppet splunk_hec --sourcetype puppet:metrics --pe_metrics > /dev/null",
-        default      => "${full_metrics_command} > /dev/null",
-      }
+      $metrics_command = "${base_metrics_command} ${metrics_shipping_command} > /dev/null"
     } else {
       $metrics_command = "${base_metrics_command} --no-print"
     }
   } else {
     $metrics_command = $override_metrics_command
   }
-  # lint:endignore
 
   $tidy_command = "${puppet_metrics_collector::scripts_dir}/metrics_tidy -d ${metrics_output_dir} -r ${retention_days}"
 
diff --git a/manifests/sar_metric.pp b/manifests/sar_metric.pp
@@ -7,6 +7,7 @@
   Integer                   $collection_frequency      = 5, # minutes
   Integer                   $polling_frequency_seconds = 1,
   String                    $metric_script_file        = 'system_metrics',
+  String                    $metrics_shipping_command  = $puppet_metrics_collector::system::metrics_shipping_command,
 ) {
 
   $metrics_output_dir = "${puppet_metrics_collector::system::output_dir}/${metrics_type}"
@@ -25,13 +26,14 @@
   $metric_script_file_path = "${puppet_metrics_collector::system::scripts_dir}/${metric_script_file}"
   $file_interval_seconds = $collection_frequency * 60
 
-  $metrics_command = join([$metric_script_file_path,
-                            " --metric_type ${metrics_type}",
-                            " --file_interval ${file_interval_seconds}",
-                            " --polling_interval ${polling_frequency_seconds}",
-                            " --metrics_dir ${puppet_metrics_collector::system::output_dir}",
-                            ' > /dev/null',
-                            ], '')
+  $base_metrics_command = join([$metric_script_file_path,
+                          "--metric_type ${metrics_type}",
+                          "--file_interval ${file_interval_seconds}",
+                          "--polling_interval ${polling_frequency_seconds}",
+                          "--metrics_dir ${puppet_metrics_collector::system::output_dir}",
+                          ], ' ')
+
+  $metrics_command = "${base_metrics_command} ${metrics_shipping_command} > /dev/null"
 
   # The hardcoded numbers with the fqdn_rand calls are to trigger the metrics_tidy 
   # command to run at a randomly selected time between 12:00 AM and 3:00 AM.
diff --git a/manifests/service/ace.pp b/manifests/service/ace.pp
@@ -14,17 +14,18 @@
   Optional[String]        $metrics_server_db_name   = $puppet_metrics_collector::metrics_server_db_name,
   ) {
   puppet_metrics_collector::pe_metric { 'ace' :
-    metric_ensure           => $metrics_ensure,
-    cron_minute             => "0/${collection_frequency}",
-    retention_days          => $retention_days,
-    hosts                   => $hosts,
-    metrics_port            => $port,
-    additional_metrics      => $extra_metrics,
-    metric_script_file      => 'puma_metrics',
-    metrics_server_type     => $metrics_server_type,
-    metrics_server_hostname => $metrics_server_hostname,
-    metrics_server_port     => $metrics_server_port,
-    metrics_server_db_name  => $metrics_server_db_name,
-    excludes                => $excludes,
+    metric_ensure            => $metrics_ensure,
+    cron_minute              => "0/${collection_frequency}",
+    retention_days           => $retention_days,
+    hosts                    => $hosts,
+    metrics_port             => $port,
+    additional_metrics       => $extra_metrics,
+    metric_script_file       => 'puma_metrics',
+    override_metrics_command => $override_metrics_command,
+    metrics_server_type      => $metrics_server_type,
+    metrics_server_hostname  => $metrics_server_hostname,
+    metrics_server_port      => $metrics_server_port,
+    metrics_server_db_name   => $metrics_server_db_name,
+    excludes                 => $excludes,
   }
 }
diff --git a/manifests/service/bolt.pp b/manifests/service/bolt.pp
@@ -14,17 +14,18 @@
   Optional[String]        $metrics_server_db_name   = $puppet_metrics_collector::metrics_server_db_name,
   ) {
   puppet_metrics_collector::pe_metric { 'bolt' :
-    metric_ensure           => $metrics_ensure,
-    cron_minute             => "0/${collection_frequency}",
-    retention_days          => $retention_days,
-    hosts                   => $hosts,
-    metrics_port            => $port,
-    additional_metrics      => $extra_metrics,
-    metric_script_file      => 'puma_metrics',
-    metrics_server_type     => $metrics_server_type,
-    metrics_server_hostname => $metrics_server_hostname,
-    metrics_server_port     => $metrics_server_port,
-    metrics_server_db_name  => $metrics_server_db_name,
-    excludes                => $excludes,
+    metric_ensure            => $metrics_ensure,
+    cron_minute              => "0/${collection_frequency}",
+    retention_days           => $retention_days,
+    hosts                    => $hosts,
+    metrics_port             => $port,
+    additional_metrics       => $extra_metrics,
+    metric_script_file       => 'puma_metrics',
+    override_metrics_command => $override_metrics_command,
+    metrics_server_type      => $metrics_server_type,
+    metrics_server_hostname  => $metrics_server_hostname,
+    metrics_server_port      => $metrics_server_port,
+    metrics_server_db_name   => $metrics_server_db_name,
+    excludes                 => $excludes,
   }
 }
diff --git a/manifests/system.pp b/manifests/system.pp
@@ -8,6 +8,10 @@
   Boolean $manage_sysstat            = false,
   Boolean $manage_vmware_tools       = false,
   String  $vmware_tools_pkg          = 'open-vm-tools',
+  Optional[Enum['influxdb', 'graphite', 'splunk_hec']] $metrics_server_type = getvar('puppet_metrics_collector::metrics_server_type'),
+  Optional[String] $metrics_server_hostname   = getvar('puppet_metrics_collector::metrics_server_hostname'),
+  Optional[Integer] $metrics_server_port      = getvar('puppet_metrics_collector::metrics_server_port'),
+  Optional[String] $metrics_server_db_name    = getvar('puppet_metrics_collector::metrics_server_db_name'),
 ) {
   $scripts_dir = "${output_dir}/scripts"
 
@@ -38,6 +42,14 @@
     refreshonly => true,
   }
 
+  $metrics_shipping_command = puppet_metrics_collector::generate_metrics_server_command(
+                                $scripts_dir,
+                                $metrics_server_type,
+                                $metrics_server_hostname,
+                                $metrics_server_db_name,
+                                $metrics_server_port
+                              )
+
   if $manage_sysstat {
     package { 'sysstat':
       ensure => $system_metrics_ensure,
diff --git a/manifests/system/cpu.pp b/manifests/system/cpu.pp
@@ -4,12 +4,14 @@
   Integer $collection_frequency      = $puppet_metrics_collector::system::collection_frequency,
   Integer $retention_days            = $puppet_metrics_collector::system::retention_days,
   Integer $polling_frequency_seconds = $puppet_metrics_collector::system::polling_frequency_seconds,
+  Optional[String] $metrics_shipping_command  = undef,
   ) {
   puppet_metrics_collector::sar_metric { 'system_cpu' :
     metric_ensure             => $metrics_ensure,
     cron_minute               => "0/${collection_frequency}",
     retention_days            => $retention_days,
     collection_frequency      => $collection_frequency,
     polling_frequency_seconds => $polling_frequency_seconds,
+    metrics_shipping_command  => $metrics_shipping_command,
   }
 }
diff --git a/manifests/system/memory.pp b/manifests/system/memory.pp
@@ -4,12 +4,14 @@
   Integer $collection_frequency      = $puppet_metrics_collector::system::collection_frequency,
   Integer $retention_days            = $puppet_metrics_collector::system::retention_days,
   Integer $polling_frequency_seconds = $puppet_metrics_collector::system::polling_frequency_seconds,
+  Optional[String] $metrics_shipping_command  = undef,
   ) {
   puppet_metrics_collector::sar_metric { 'system_memory' :
     metric_ensure             => $metrics_ensure,
     cron_minute               => "0/${collection_frequency}",
     retention_days            => $retention_days,
     collection_frequency      => $collection_frequency,
     polling_frequency_seconds => $polling_frequency_seconds,
+    metrics_shipping_command  => $metrics_shipping_command,
   }
 }
diff --git a/manifests/system/postgres.pp b/manifests/system/postgres.pp
@@ -11,6 +11,7 @@
   String  $metrics_ensure            = $puppet_metrics_collector::system::system_metrics_ensure,
   Integer $collection_frequency      = $puppet_metrics_collector::system::collection_frequency,
   Integer $retention_days            = $puppet_metrics_collector::system::retention_days,
+  String  $metrics_shipping_command  = $puppet_metrics_collector::system::metrics_shipping_command,
 ) {
   $metrics_output_dir = "${puppet_metrics_collector::system::output_dir}/postgres"
   $metrics_output_dir_ensure = $metrics_ensure ? {
@@ -26,6 +27,7 @@
 
   $metrics_command = ["${puppet_metrics_collector::system::scripts_dir}/psql_metrics",
                       '--output_dir', $metrics_output_dir,
+                      $metrics_shipping_command,
                       '> /dev/null'].join(' ')
 
   $tidy_command = "${puppet_metrics_collector::system::scripts_dir}/metrics_tidy -d ${metrics_output_dir} -r ${retention_days}"
diff --git a/manifests/system/processes.pp b/manifests/system/processes.pp
diff --git a/manifests/system/vmware.pp b/manifests/system/vmware.pp
diff --git a/spec/classes/puppet_metrics_collector_system_spec.rb b/spec/classes/puppet_metrics_collector_system_spec.rb
diff --git a/spec/functions/generate_metrics_server_command_spec.rb b/spec/functions/generate_metrics_server_command_spec.rb
