added custom tags #27

Author: adejoux

docs/config.toml

@@ -3,6 +3,7 @@ title = "nmon2influxdb"
 theme = "darkdoc"
 metadataformat = "yaml"
 pygmentsUseClasses = true
+PygmentsCodeFences = true
 
 [params]
 	# General information

docs/content/configuration/file.md

@@ -72,7 +72,7 @@ stats_host=""
 
 If you are always querying the same host or applying the same timeframe to your queries you can setup here this values.
 
-##data retention
+## data retention
 
 By default, data are kept indefinitely in InfluxDB. It's possible to change it to have data expiration.
 
@@ -85,3 +85,53 @@ This value is updated each time a import is done.
 All data older than what are specified in the retention policy are not kept.
 
 **Note:** it's the timestamp associated with data which matters. If you load data from one year ago and you have a retention policy of 30 days, you will not see the data.
+
+## custom tags
+
+Starting with version [2.1.0](/210_version_released), it's possible to add custom tags to your data by adding a input section in the configuration file:
+
+``` toml
+[[input]]
+  Measurement="PartitionProcessor"
+  Name="partition"
+  Match="adxlpar"
+  [[input.tag]]
+    Name="datacenter"
+    Value="DC1"
+```
+Custom tags are added at import time.
+It will add a tag named **datacenter** with value **DC1** if the tag **partition** in the measurement **PartitionProcessor** match the regular expression **adxlpar**.
+
+Attribute's description:
+
+  * **Measurement**: it's the measurement where an additional tag could be added
+  * **Name**: name of the tag to check
+  * **Match**: the regular expression used to check the tag value. No need to put the regular expression between '/' characters.
+
+It's possible to add multiple tags for the same data:
+
+
+``` toml
+[[input]]
+  Measurement="PartitionProcessor"
+  Name="partition"
+  Match="adxlpar"
+  [[input.tag]]
+    Name="datacenter"
+    Value="DC1"
+  [[input.tag]]
+    Name="group"
+    Value="adx"
+```
+
+If you want to match the exact name, use regular expression syntax (^ and $):
+
+``` toml
+[[input]]
+  Measurement="PartitionProcessor"
+  Name="partition"
+  Match="^adxlpar1$"
+  [[input.tag]]
+    Name="datacenter"
+    Value="DC1"
+```

docs/content/news/2.1.0_version_released.md

@@ -0,0 +1,39 @@
+---
+date: 2016-11-30T13:01:53+01:00
+title: 2.1.0 version released
+type: "news"
+url: "/210_version_released"
+---
+New version released adding custom tags.
+
+<!--more-->
+
+# Custom Tags
+
+add custom tags to your data by adding a input section in the configuration file:
+
+``` toml
+[[input]]
+  Measurement="PartitionProcessor"
+  Name="partition"
+  Match="adxlpar"
+  [[input.tag]]
+    Name="datacenter"
+    Value="DC1"
+```
+Custom tags are added at import time.
+It will add a tag named **datacenter** with value **DC1** if the tag **partition** in the measurement **PartitionProcessor** match the regular expression **adxlpar**.
+
+Attribute's description:
+
+  * **Measurement**: it's the measurement where an additional tag could be added
+  * **Name**: name of the tag to check
+  * **Match**: the regular expression used to check the tag value. No need to put the regular expression between '/' characters.
+
+More informations on the [configuration file page](/configuration/file/).
+
+# enhancements
+
+  * added **serial** tag to nmon files to permit grouping by managed system.
+  * added information message if retention policy is set.
+  * added error message on configuration file syntax error.

hmc/hmc.go

@@ -36,6 +36,7 @@ type HMC struct {
 	Debug               bool
 	ManagedSystemOnly   bool
 	Samples             int
+	TagParsers          nmon2influxdblib.TagParsers
 }
 
 // Point is a struct to simplify InfluxDB point creation
@@ -74,6 +75,11 @@ func NewHMC(c *cli.Context) *HMC {
 	hmc.Samples = config.HMCSamples
 	hmc.Debug = config.Debug
 	hmc.FilterManagedSystem = config.HMCManagedSystem
+
+	if len(config.Inputs) > 0 {
+		//Build tag parsing
+		hmc.TagParsers = nmon2influxdblib.ParseInputs(config.Inputs)
+	}
 	hmcURL := fmt.Sprintf("https://"+"%s"+":12443", config.HMCServer)
 	//initialize new http session
 	hmc.Session = NewSession(config.HMCUser, config.HMCPassword, hmcURL)
@@ -135,6 +141,17 @@ func (hmc *HMC) AddPoint(point Point) {
 		tags["partition"] = hmc.GlobalPoint.Partition
 	}
 	field := map[string]interface{}{"value": value}
+
+	// Checking additional tagging
+	for key, value := range tags {
+		if _, ok := hmc.TagParsers[point.Name][key]; ok {
+			for _, tagParser := range hmc.TagParsers[point.Name][key] {
+				if tagParser.Regexp.MatchString(value) {
+					tags[tagParser.Name] = tagParser.Value
+				}
+			}
+		}
+	}
 	hmc.InfluxDB.AddPoint(point.Name, hmc.GlobalPoint.Timestamp, field, tags)
 }
 

main.go

@@ -55,7 +55,7 @@ func main() {
 	app := cli.NewApp()
 	app.Name = "nmon2influxdb"
 	app.Usage = "upload NMON stats to InfluxDB database"
-	app.Version = "2.0.1"
+	app.Version = "2.1.0"
 	app.Commands = []cli.Command{
 		{
 			Name:  "import",

nmon/import.go

@@ -19,6 +19,7 @@ import (
 )
 
 var hostRegexp = regexp.MustCompile(`^AAA,host,(\S+)`)
+var serialRegexp = regexp.MustCompile(`^AAA,SerialNumber,(\S+)`)
 var osRegexp = regexp.MustCompile(`^AAA,.*(Linux|AIX)`)
 var timeRegexp = regexp.MustCompile(`^ZZZZ,([^,]+),(.*)$`)
 var intervalRegexp = regexp.MustCompile(`^AAA,interval,(\d+)`)
@@ -51,20 +52,26 @@ func Import(c *cli.Context) {
 	nmonFiles := new(nmon2influxdblib.Files)
 	nmonFiles.Parse(c.Args(), config.ImportSSHUser, config.ImportSSHKey)
 
+	tagParsers := nmon2influxdblib.ParseInputs(config.Inputs)
+
+	var userSkipRegexp *regexp.Regexp
+	if len(config.ImportSkipMetrics) > 0 {
+		skipped := strings.Replace(config.ImportSkipMetrics, ",", "|", -1)
+		userSkipRegexp = regexp.MustCompile(skipped)
+	}
+
 	for _, nmonFile := range nmonFiles.Valid() {
 		var count int64
 		count = 0
 		nmon := InitNmon(config, nmonFile)
 
-		lines := nmonFile.Content()
-
-		var userSkipRegexp *regexp.Regexp
-
-		if len(config.ImportSkipMetrics) > 0 {
-			skipped := strings.Replace(config.ImportSkipMetrics, ",", "|", -1)
-			userSkipRegexp = regexp.MustCompile(skipped)
+		if len(config.Inputs) > 0 {
+			//Build tag parsing
+			nmon.TagParsers = tagParsers
 		}
 
+		lines := nmonFile.Content()
+
 		var last string
 		filters := new(influxdbclient.Filters)
 		filters.Add("file", path.Base(nmonFile.Name), "text")
@@ -156,9 +163,19 @@ func Import(c *cli.Context) {
 						measurement = nameRegexp.ReplaceAllString(name, "")
 					}
 
+					// Checking additional tagging
+					for key, value := range tags {
+						if _, ok := nmon.TagParsers[measurement][key]; ok {
+							for _, tagParser := range nmon.TagParsers[measurement][key] {
+								if tagParser.Regexp.MatchString(value) {
+									tags[tagParser.Name] = tagParser.Value
+								}
+							}
+						}
+					}
 					influxdb.AddPoint(measurement, timestamp, field, tags)
 
-					if influxdb.PointsCount() == 10000 {
+					if influxdb.PointsCount() == 5000 {
 						err = influxdb.WritePoints()
 						nmon2influxdblib.CheckError(err)
 						count += influxdb.PointsCount()
@@ -204,6 +221,10 @@ func Import(c *cli.Context) {
 
 					tags := map[string]string{"host": nmon.Hostname, "name": column, "pid": elems[1], "command": elems[13], "wlm": wlmclass}
 
+					if len(nmon.Serial) > 0 {
+						tags["serial"] = nmon.Serial
+					}
+
 					// try to convert string to integer
 					converted, parseErr := strconv.ParseFloat(value, 64)
 					if parseErr != nil {

nmon/nmon.go

@@ -20,6 +20,7 @@ import (
 // Nmon structure used to manage nmon files
 type Nmon struct {
 	Hostname    string
+	Serial      string
 	OS          string
 	TimeStamps  map[string]string
 	TextContent string
@@ -29,6 +30,7 @@ type Nmon struct {
 	starttime   time.Time
 	stoptime    time.Time
 	Location    *time.Location
+	TagParsers  nmon2influxdblib.TagParsers
 }
 
 // DataSerie structure contains the columns and points to insert in InfluxDB
@@ -142,6 +144,12 @@ func InitNmon(config *nmon2influxdblib.Config, nmonFile nmon2influxdblib.File) (
 			continue
 		}
 
+		if serialRegexp.MatchString(line) {
+			matched := serialRegexp.FindStringSubmatch(line)
+			nmon.Serial = strings.ToLower(matched[1])
+			continue
+		}
+
 		if osRegexp.MatchString(line) {
 			matched := osRegexp.FindStringSubmatch(line)
 			nmon.OS = strings.ToLower(matched[1])

nmon2influxdblib/config.go

@@ -59,6 +59,18 @@ type Config struct {
 	Metric               string `toml:"metric,omitempty"`
 	ListFilter           string `toml:",omitempty"`
 	ListHost             string `toml:",omitempty"`
+	Inputs               Inputs `toml:"input,omitempty"`
+}
+
+// Inputs allows to put multiple input in the configuration file
+type Inputs []Input
+
+// Input specify how to apply new filters
+type Input struct {
+	Measurement string
+	Name        string
+	Match       string
+	Tags        Tags `toml:"tag"`
 }
 
 // InitConfig setup initial configuration with sane values
@@ -157,9 +169,9 @@ func (config *Config) LoadCfgFile() {
 	}
 
 	if err := toml.Unmarshal(buf, &config); err != nil {
-		CheckError(err)
+		fmt.Printf("syntax error in configuration file: %s \n", err.Error())
+		os.Exit(1)
 	}
-
 }
 
 // AddDashboardParams initialize default parameters for dashboard
@@ -256,6 +268,7 @@ func (config *Config) GetDB(dbType string) *influxdbclient.InfluxDB {
 	influxdb := config.ConnectDB(db)
 
 	if exist, _ := influxdb.ExistDB(db); exist != true {
+		fmt.Printf("Creating InfluxDB database %s\n", db)
 		_, createErr := influxdb.CreateDB(db)
 		CheckError(createErr)
 	}
@@ -265,6 +278,7 @@ func (config *Config) GetDB(dbType string) *influxdbclient.InfluxDB {
 		// Get default retention policy name
 		policyName, policyErr := influxdb.GetDefaultRetentionPolicy()
 		CheckError(policyErr)
+		fmt.Printf("Updating  %s retention policy to keep only the last %s days. Timestamp based.\n", policyName, retention)
 		_, err := influxdb.UpdateRetentionPolicy(policyName, retention, true)
 		CheckError(err)
 	}

nmon2influxdblib/parser.go

@@ -0,0 +1,50 @@
+// nmon2influxdb
+// author: adejoux@djouxtech.net
+
+package nmon2influxdblib
+
+import (
+	"fmt"
+	"regexp"
+)
+
+//Tags array
+type Tags []Tag
+
+// Tag is a struct to store additional tags
+type Tag struct {
+	Name   string
+	Value  string
+	Regexp *regexp.Regexp `toml:",skip"`
+}
+
+// TagParsers access struct : TagParsers[mesurement][tag name]
+type TagParsers map[string]map[string]Tags
+
+// ParseInputs process user inputs and compile regular expressions
+func ParseInputs(inputs Inputs) TagParsers {
+	tagParsers := make(TagParsers)
+
+	for _, input := range inputs {
+		tagRegexp, RegCompErr := regexp.Compile(input.Match)
+		if RegCompErr != nil {
+			fmt.Printf("could not compile Config Input match parameter %s\n", input.Match)
+		}
+
+		// Intialize if empty struct
+		if _, ok := tagParsers[input.Measurement]; !ok {
+			tagParsers[input.Measurement] = make(map[string]Tags)
+		}
+
+		tagger := tagParsers[input.Measurement][input.Name]
+
+		for _, tag := range input.Tags {
+			tag.Regexp = tagRegexp
+			tagger = append(tagger, tag)
+		}
+
+		tagParsers[input.Measurement][input.Name] = tagger
+
+	}
+	return tagParsers
+}