(SLV-672) Add system class to manage system metric

RandellP · RandellP · commit bec1cda36d96 · 2019-11-22T16:17:39.000-08:00
The new system class is very similar to the init.pp... this is to keep the two seperate during development.
They may get merged in teh future.  But the work flow is such that the master would have puppet metrics collector and sysystem included, while the compilers would only have system.  To merge them would require a clean way to turn off all pe metrics for the compilers.

Also added system_cpu and system_memory to use sar_metric to gather cpu and memory metrics.
diff --git a/README.md b/README.md
@@ -37,6 +37,19 @@ node 'master.example.com' {
 }
 ```
 
+Optionally, you can also gather some basic system metrics.  Unlike the service metrics, this has to be enabled on each host you want metrics from, and the resulting data will be only on that host.  Do not include the top level puppet_metrics_collector on anything other than the master as it will collect the same data as the one on the master.  This functionality depends on sysstat.
+
+```
+node 'master.example.com' {
+  include puppet_metrics_collector
+  include puppet_metrics_collector::system
+}
+
+node 'compilerA.example.com', 'compilerB.example.com,' {
+  include puppet_metrics_collector::system
+}
+``` 
+
 ### Configuration
 
 This module automatically configures the hosts it queries by querying PuppetDB for PE Infrastructure Hosts. If there is an error with automatic configuration of hosts, refer to [Manual Configuration of Hosts](#manual-configuration-of-hosts).
diff --git a/files/generate_system_metrics b/files/generate_system_metrics
@@ -16,7 +16,7 @@ require "fileutils"
 # sar_metric.pp will setup a cron job to run this similar to how pe_metric runs tk_metrics
 #
 # Example execution
-# generate_system_metrics --metric_type cpu --file_interval 300 --polling_interval 1
+# generate_system_metrics --metric_type system_cpu --file_interval 300 --polling_interval 1
 #                         --metrics_dir /opt/puppetlabs/puppet-metrics-collector
 
 # General namespace for SystemMetrics module
@@ -27,7 +27,7 @@ module SystemMetrics
   #
   # @attr [integer] polling_interval Time in seconds between calls to poll the system for data.
   # @attr [integer] file_interval Time in seconds between the creation of each output file.
-  # @attr [string] metric_type cpu|memory
+  # @attr [string] metric_type system_cpu|system_memory
   # @attr [string] metrics_dir The puppet_metrics_collector output directory.
   # @attr [boolean] verbose Verbose output
   # @attr [string] hostname Name of the host the metrics are from. In directory name and json file.
@@ -40,7 +40,7 @@ module SystemMetrics
     #
     # @param [integer] polling_interval Time in seconds between calls to poll the system for data.
     # @param [integer] file_interval Time in seconds between the creation of each output file.
-    # @param [string] metric_type cpu|memory
+    # @param [string] metric_type system_cpu|system_memory
     # @param [string] metrics_dir The puppet_metrics_collector output directory.
     # @param [boolean] verbose Verbose output
     #
@@ -65,7 +65,7 @@ module SystemMetrics
     def run_sar
       times_to_poll = (@file_interval / @polling_interval).round
       # sar inputs are polling interval and how many times to poll
-      comm_flags = " -r" if @metric_type =~ /memory/
+      comm_flags = " -r" if @metric_type =~ /system_memory/
       comm = "sar #{comm_flags} #{@polling_interval} #{times_to_poll}"
       puts "sar command is: #{comm}" if @verbose
       %x[#{comm}]
@@ -83,7 +83,7 @@ module SystemMetrics
     def parse_sar_output(sar_output)
       sar_output_arr = sar_output.split(/\n+|\r+/).reject(&:empty?).map { |line| line.split }
 
-      if ( @metric_type == "memory")
+      if ( @metric_type == "system_memory")
         unique_header_str = "%memused"
       else
         unique_header_str = "%user"
@@ -142,7 +142,7 @@ module SystemMetrics
     # @return [void]
     def create_file(json_dataset, time_stamp_obj)
       filename = time_stamp_obj.utc.strftime('%Y%m%dT%H%M%SZ') + '.json'
-      dirname = "#{@metrics_dir}/system_#{@metric_type}/#{@hostname}"
+      dirname = "#{@metrics_dir}/#{@metric_type}/#{@hostname}"
       file_path = "#{dirname}/#{filename}"
       FileUtils.mkdir_p(dirname) unless File.directory?(dirname)
       puts "Creating json file: #{file_path}" if @verbose
@@ -167,10 +167,10 @@ end
 
 if $PROGRAM_NAME == __FILE__
 
-  VALID_METRIC_TYPES = %w[cpu memory]
+  VALID_METRIC_TYPES = %w[system_cpu system_memory]
   FILE_INTERVAL_DEFAULT = 60 * 5
   POLLING_INTERVAL_DEFAULT = 1
-  METRIC_TYPE_DEFAULT = "cpu"
+  METRIC_TYPE_DEFAULT = "system_cpu"
   METRICS_DIR_DEFAULT = "/opt/puppetlabs/puppet-metrics-collector"
 
   DESCRIPTION = <<-DESCRIPTION
diff --git a/manifests/sar_metric.pp b/manifests/sar_metric.pp
@@ -0,0 +1,60 @@
+define puppet_metrics_collector::sar_metric (
+  String                    $output_dir,
+  String                    $scripts_dir,
+  Enum['absent', 'present'] $metric_ensure = 'present',
+  String                    $metrics_type = $title,
+  String                    $cron_minute = '*/5',
+  Integer                   $retention_days = 90,
+  Integer                   $polling_frequency_seconds = 1,
+  Integer                   $collection_frequency = 5, #minutes
+  String                    $metric_script_file = 'generate_system_metrics',
+) {
+
+  $metrics_output_dir = "${output_dir}/${metrics_type}"
+
+  $_metric_ensure = $metric_ensure ? {
+    'present' => directory,
+    'absent'  => absent,
+  }
+
+  file { $metrics_output_dir :
+    ensure => $_metric_ensure,
+  }
+
+  $script_file_name = "${scripts_dir}/${metric_script_file}"
+  $file_interval_seconds = $collection_frequency * 60
+
+  $metrics_command = join(["${script_file_name} --metric_type ${metrics_type}",
+                            " --file_interval ${file_interval_seconds}",
+                            " --polling_interval ${polling_frequency_seconds}",
+                            " --metrics_dir ${output_dir}"], '')
+
+  cron { "${metrics_type}_metrics_collection" :
+    ensure  => $metric_ensure,
+    command => $metrics_command,
+    user    => 'root',
+    minute  => $cron_minute,
+  }
+
+  $metrics_tidy_script_path = "${scripts_dir}/${metrics_type}_metrics_tidy"
+
+  file { $metrics_tidy_script_path :
+    ensure  => $metric_ensure,
+    mode    => '0744',
+    content => epp('puppet_metrics_collector/tidy_cron.epp', {
+      'metrics_output_dir' => $metrics_output_dir,
+      'metrics_type'       => $metrics_type,
+      'retention_days'     => $retention_days,
+    }),
+  }
+
+  # The hardcoded numbers with the fqdn call are to trigger the tidy to run at a randomly selected
+  # time between 12:00 AM and 3:00 AM
+  cron { "${metrics_type}_metrics_tidy" :
+    ensure  => $metric_ensure,
+    user    => 'root',
+    hour    => fqdn_rand(3, $metrics_type),
+    minute  => (5 * fqdn_rand(11, $metrics_type)),
+    command => $metrics_tidy_script_path
+  }
+}
diff --git a/manifests/system.pp b/manifests/system.pp
@@ -0,0 +1,55 @@
+class puppet_metrics_collector::system (
+  String        $output_dir                    = '/opt/puppetlabs/puppet-metrics-collector',
+  Integer       $collection_frequency          = 5, #minutes
+  Integer       $polling_frequency_seconds     = 1,
+  Integer       $retention_days                = 90,
+  String        $system_metrics_ensure         = present,
+  Boolean       $symlink_puppet_metrics_collector = true,
+  Boolean       $manage_sysstat = true,
+) {
+  $scripts_dir = "${output_dir}/scripts"
+  $bin_dir     = "${output_dir}/bin"
+
+  #assume if output is defined, all of the rest will be too as the init.pp must be in use
+  #and thus we don't need to redefine these
+  if ! defined(File[$output_dir]) {
+    file { [ $output_dir, $scripts_dir, $bin_dir]:
+      ensure => directory,
+    }
+
+    file { "${bin_dir}/puppet-metrics-collector":
+      ensure  => file,
+      owner   => 'root',
+      group   => 'root',
+      mode    => '0755',
+      content => epp('puppet_metrics_collector/puppet-metrics-collector.epp', {
+        'output_dir' => $output_dir,
+      }),
+    }
+
+    $symlink_ensure = $symlink_puppet_metrics_collector ? {
+      false => 'absent',
+      true  => 'symlink',
+    }
+
+    file { '/opt/puppetlabs/bin/puppet-metrics-collector':
+      ensure => $symlink_ensure,
+      target => "${bin_dir}/puppet-metrics-collector",
+    }
+  }
+
+  file { "${scripts_dir}/generate_system_metrics":
+    ensure => present,
+    mode   => '0755',
+    source => 'puppet:///modules/puppet_metrics_collector/generate_system_metrics'
+  }
+
+  if $manage_sysstat {
+    package { 'sysstat':
+      ensure => installed,
+    }
+  }
+
+  include puppet_metrics_collector::system_cpu
+  include puppet_metrics_collector::system_memory
+}
diff --git a/manifests/system_cpu.pp b/manifests/system_cpu.pp
@@ -0,0 +1,19 @@
+class puppet_metrics_collector::system_cpu (
+  Integer $collection_frequency      = $puppet_metrics_collector::system::collection_frequency,
+  Integer $polling_frequency_seconds = $puppet_metrics_collector::system::polling_frequency_seconds,
+  Integer $retention_days            = $puppet_metrics_collector::system::retention_days,
+  String  $metrics_ensure            = $puppet_metrics_collector::system::system_metrics_ensure,
+  ) {
+  Puppet_metrics_collector::Sar_metric {
+    output_dir                => $puppet_metrics_collector::system::output_dir,
+    scripts_dir               => $puppet_metrics_collector::system::scripts_dir,
+    cron_minute               => "*/${collection_frequency}",
+    collection_frequency      => $collection_frequency,
+    polling_frequency_seconds => $polling_frequency_seconds,
+    retention_days            => $retention_days,
+  }
+
+  puppet_metrics_collector::sar_metric { 'system_cpu' :
+    metric_ensure             => $metrics_ensure,
+  }
+}
diff --git a/manifests/system_memory.pp b/manifests/system_memory.pp
@@ -0,0 +1,19 @@
+class puppet_metrics_collector::system_memory (
+  Integer $collection_frequency      = $puppet_metrics_collector::system::collection_frequency,
+  Integer $polling_frequency_seconds = $puppet_metrics_collector::system::polling_frequency_seconds,
+  Integer $retention_days            = $puppet_metrics_collector::system::retention_days,
+  String  $metrics_ensure            = $puppet_metrics_collector::system::system_metrics_ensure,
+  ) {
+  Puppet_metrics_collector::Sar_metric {
+    output_dir                => $puppet_metrics_collector::system::output_dir,
+    scripts_dir               => $puppet_metrics_collector::system::scripts_dir,
+    cron_minute               => "*/${collection_frequency}",
+    collection_frequency      => $collection_frequency,
+    polling_frequency_seconds => $polling_frequency_seconds,
+    retention_days            => $retention_days,
+  }
+
+  puppet_metrics_collector::sar_metric { 'system_memory' :
+    metric_ensure             => $metrics_ensure,
+  }
+}
