release-utils improvements...

share/release-utils/Dockerfile

@@ -1,4 +1,4 @@
-FROM dpb587/logsearch-logstash-dev:v19
+FROM dpb587/logsearch-logstash-dev
 ADD . /usr/local/logsearch-shipper-release-utils
 ENTRYPOINT /usr/local/logsearch-shipper-release-utils/bin/entrypoint
 ENV RELEASE_DIR release

share/release-utils/README.md

@@ -2,20 +2,53 @@ The goal of these conventions is to make it easier for releases to expose their
 
  0. Annotate your logs with details about how they're formatted.
  0. Include those details alongside your release version tarballs.
- 0. Add hooks to your deployment processes to extract the configuration and ship them to your logsearch deployment.
- 0. Deploy your actual release (colocated with logsearch-shipper).
+ 0. Use hooks in your deployment processes to extract the configuration and update your logsearch deployment.
+
+This stuff is an idea in progress...
 
 
 ## Release
 
-These scripts expect the following conventions...
+These scripts use and expect the following conventions...
+
+### Logs
+
+For inspection...
+
+ * every log format MUST be defined in `jobs/{job-name}/logsearch/logs/{log-reference-name}/` where:
+    * the `{log-reference-name}` SHOULD be `{log-name}-{version}`
+    * there SHOULD be `logstash-filters.conf` which SHOULD describe how to parse every line, and
+    * there SHOULD be `elasticsearch-mappings.json` which SHOULD describe all the elasticsearch mappings for every extracted field, and
+    * there SHOULD be `expected.testdata` listing sample log lines and their parsed values, and
+    * there MAY be custom `*spec.rb` files containing more extensive tests
+ * every log format MUST have a fully unique name to distinguish between releases and jobs with similar, but different formats
+    * log formats SHOULD be named with the template `{release-name}-{job-name}-{log-name}-{version}`
+    * log formats MAY use a custom name by writing it to `jobs/{job-name}/logsearch/logs/{log-reference-name}/name`
+ * if the log format changes or mappings were defined incorrectly, versioning MUST be incremented
+
+### Dashboards
+
+For visualizing...
+
+ * dashboards MUST be defined as job-specific, or deployment-specific in the following directories:
+    * job-specific dashboards MUST be defined in `jobs/{job-name}/logsearch/dashboards/{dashboard-name}/`
+    * deployment dashboards MUST be defined in `src/logsearch/dashboards/{dashboard-name}/`
+ * the dashboard directories SHOULD be structured:
+    * with a `README.md` file which:
+       * MUST provide an brief abstract describing the dashboard
+       * SHOULD be followed by helpful information about the dashboard for end users
+    * with a `thumbnail.png` file providing a preview of the dashboard
+    * at least one interface-specific file MUST exist which conforms to its respective dashboard configuration format
+ * interface-specific dashboards MAY assume interpolation for the following variables which SHOULD be implemented by installation tools:
+    * `director`
+    * `deployment`
+    * `job` (`{job_name}/{job_index}`)
+    * `job_name`
+    * `job_index`
+
+### Monitors
 
- * every log format MUST be named by `{release-name}-{job-name}-{log-name}-{version}`
- * every log format MUST be documented in `jobs/{job-name}/logsearch/logs/{log-name}-{version}/` where:
-    * there MUST be `logstash-filters.conf` which SHOULD describe how to parse every line, and
-    * there SHOULD be `elasticsearch-mappings.json` which SHOULD describe all the elasticsearch mapping for every extracted field, and
-    * there SHOULD be `expected.testdata` listing sample log lines and their parsed values
- * if the log format changes or mappings were defined incorrectly, the version MUST be incremented
+For avoidance...
 
 
 ## Docker
@@ -28,7 +61,7 @@ All of these scripts can be compiled into a Docker image which is based on the c
 
 ## Concourse
 
-There's a sample task you can use in your build pipelines called [`append-logsearch-config`](./ci/tasks/append-logsearch-config). It can be used to test and append a `logsearch-config` directory to the release tarballs which makes it easier to reference your configurations; for example...
+There's a sample task you can use in your build pipelines called [`append-logsearch-config`](./ci/tasks/append-logsearch-config). It can be used to test and append a `logsearch-config` directory to the release tarballs which makes it easier to reference your configurations. For example...
 
     $ RELEASE_NAME=myrelease ; RELEASE_VERSION=20 ; RELEASE_URL=https://...
     $ RELEASE_FILE="${RELEASE_NAME}-${RELEASE_VERSION}.tgz"

share/release-utils/bin/build

@@ -16,7 +16,7 @@ mkdir logsearch-config/logs
 echo '==> Generating logstash filters...'
 
 mkdir logsearch-config/logs/logstash-filters
-"${EXECDIR}/logs/logstash-filters/enumerate" --active > logsearch-config/logs/logstash-filters.txt
+"${EXECDIR}/logs/enumerate" logstash-filters.conf --active > logsearch-config/logs/logstash-filters.txt
 cat logsearch-config/logs/logstash-filters.txt | "${EXECDIR}/logs/logstash-filters/copy" logsearch-config/logs/logstash-filters
 cat logsearch-config/logs/logstash-filters.txt | "${EXECDIR}/logs/logstash-filters/collect" > logsearch-config/logs/logstash-filters.conf
 rm logsearch-config/logs/logstash-filters.txt
@@ -25,9 +25,9 @@ rm logsearch-config/logs/logstash-filters.txt
 echo '==> Generating elasticsearch mappings...'
 
 mkdir logsearch-config/logs/elasticsearch-mappings
-"${EXECDIR}/logs/elasticsearch-mappings/enumerate" --active > logsearch-config/logs/elasticsearch-mappings.txt
+"${EXECDIR}/logs/enumerate" elasticsearch-mapping.json --active > logsearch-config/logs/elasticsearch-mappings.txt
 cat logsearch-config/logs/elasticsearch-mappings.txt | "${EXECDIR}/logs/elasticsearch-mappings/copy" logsearch-config/logs/elasticsearch-mappings
-cat logsearch-config/logs/elasticsearch-mappings.txt | "${EXECDIR}/logs/elasticsearch-mappings/collect" > logsearch-config/logs/elasticsearch-mappings.conf
+cat logsearch-config/logs/elasticsearch-mappings.txt | "${EXECDIR}/logs/elasticsearch-mappings/collect" > logsearch-config/logs/elasticsearch-mappings.json
 rm logsearch-config/logs/elasticsearch-mappings.txt
 
 

share/release-utils/logs/enumerate

@@ -0,0 +1,105 @@
+#!/usr/bin/env ruby
+
+require 'yaml'
+
+release_name = YAML.load_file('config/final.yml')['final_name']
+
+#
+# naive command line parsing
+#
+
+filetype = nil
+verbosity = 2
+showmode = '--active'
+aliases = {}
+
+confs = {}
+globalconfs = []
+showconfs = []
+
+ARGV.each do | argv |
+  if [ '--active', '--all' ].include? argv
+    showmode = argv
+  elsif '-q' == argv
+    verbosity -= 1
+  elsif argv.include? '='
+    split = argv.split('=', 2)
+
+    if '' == split[1]
+      showconfs.push split[0]
+    elsif File.exists? split[1]
+      confs[split[0]] = File.realpath(split[1])
+    else
+      $stderr.puts "WARNING: #{split[1]} passed in argument but not found on filesystem"
+    end
+  else
+    filetype = argv
+  end
+end
+
+if nil == filetype
+  $stderr.puts "ERROR: a file type to enumerate is required as an argument" if verbosity > 0
+
+  exit 1
+end
+
+#
+# enumerate all the available files
+#
+
+Dir.glob("jobs/**/logsearch/logs/*/#{filetype}")
+.concat(Dir.glob("src/logsearch-config/logs/*/#{filetype}"))
+.each do | joblogconf |
+  log_name = File.basename(File.dirname(joblogconf))
+  job_name = File.basename(File.dirname(File.dirname(File.dirname(File.dirname(joblogconf)))))
+
+  name = "#{release_name}-#{job_name}-#{log_name}"
+  overridename = "#{File.dirname(joblogconf)}/name"
+
+  if File.exists? overridename
+    name = File.read(overridename).strip
+  end
+
+  if !confs.has_key?(name)
+    confs[name] = File.realpath(joblogconf)
+    globalconfs.push(name) if joblogconf.match(/^src/)
+  end
+end
+
+#
+# now filter reduce to what they want to see
+#
+
+if '--active' == showmode
+  Dir.glob("jobs/**/templates/logsearch/logs.yml").each do | logfile |
+    job_name = File.basename(File.dirname(File.dirname(File.dirname(logfile))))
+
+    logyaml = YAML.load_file(logfile)
+
+    logyaml['files'].each do | joblogfile, joblogconfig |
+      if nil == joblogconfig
+        next
+      elsif !joblogconfig.has_key? 'fields' or !joblogconfig['fields'].has_key? 'type'
+        $stderr.puts "WARNING: #{joblogfile} from job #{job_name} does not define a type field" if verbosity > 1
+
+        next
+      elsif !confs.has_key? joblogconfig['fields']['type']
+        $stderr.puts "WARNING: #{joblogfile} from job #{job_name} is typed #{joblogconfig['fields']['type']} which is not defined here" if verbosity > 1
+
+        next
+      end
+
+      showconfs.push joblogconfig['fields']['type']
+    end
+  end
+else
+  showconfs = confs.keys
+end
+
+#
+# show them
+#
+
+showconfs.each do | name |
+  $stdout.puts "#{name}\t#{confs[name]}"
+end

share/release-utils/logs/logstash-filters/spec.rb

@@ -3,7 +3,9 @@
 require 'spec_helper'
 require 'json'
 
-Dir.glob("#{ENV['RELEASE_DIR']}/jobs/**/logsearch/logs/*/expected.testdata").each do | path |
+Dir.glob("#{ENV['RELEASE_DIR']}/jobs/**/logsearch/logs/*/expected.testdata")
+.concat(Dir.glob("#{ENV['RELEASE_DIR']}/src/logsearch-config/logs/*/expected.testdata"))
+.each do | path |
   describe File.dirname(path) do
     config 'filter {' + File.read("#{File.dirname(path)}/logstash-filters.conf") + '}'