Adding stackdriver exporter

Author: Xuewei Zhang

.travis.yml

@@ -27,3 +27,5 @@ script:
  - BUILD_TAGS="disable_system_log_monitor" make test
  - make clean && BUILD_TAGS="disable_system_stats_monitor" make
  - BUILD_TAGS="disable_system_stats_monitor" make test
+ - make clean && BUILD_TAGS="disable_stackdriver_exporter" make
+ - BUILD_TAGS="disable_stackdriver_exporter" make test

Makefile

@@ -41,7 +41,7 @@ PKG:=k8s.io/node-problem-detector
 PKG_SOURCES:=$(shell find pkg cmd -name '*.go')
 
 # TARBALL is the name of release tar. Include binary version by default.
-TARBALL:=node-problem-detector-$(VERSION).tar.gz
+TARBALL?=node-problem-detector-$(VERSION).tar.gz
 
 # IMAGE is the image name of the node problem detector container image.
 IMAGE:=$(REGISTRY)/node-problem-detector:$(TAG)

README.md

@@ -69,27 +69,44 @@ List of supported problem daemons:
 # Exporter
 
 An exporter is a component of node-problem-detector. It reports node problems and/or metrics to
-certain back end (e.g. Kubernetes API server, or Prometheus scrape endpoint).
+certain back end. Some of them can be disable at compile time using a build tag. List of supported exporters:
+
+| Exporter |Description | Disabling Build Tag |
+|----------|:-----------|:--------------------|
+| Kubernetes exporter | Kubernetes exporter reports node problems to Kubernetes API server: temporary problems get reported as Events, and permanent problems get reported as Node Conditions. | 
+| Prometheus exporter | Prometheus exporter reports node problems and metrics locally as Prometheus metrics | 
+| [Stackdriver exporter](https://github.com/kubernetes/node-problem-detector/blob/master/config/exporter/stackdriver-exporter.json) | Stackdriver exporter reports node problems and metrics to Stackdriver Monitoring API. | disable_stackdriver_exporter
 
 # Usage
 
 ## Flags
 
 * `--version`: Print current version of node-problem-detector.
-* `--address`: The address to bind the node problem detector server.
-* `--port`: The port to bind the node problem detector server. Use 0 to disable.
+* `--hostname-override`: A customized node name used for node-problem-detector to update conditions and emit events. node-problem-detector gets node name first from `hostname-override`, then `NODE_NAME` environment variable and finally fall back to `os.Hostname`.
+
+#### For System Log Monitor
+
 * `--config.system-log-monitor`: List of paths to system log monitor configuration files, comma separated, e.g.
   [config/kernel-monitor.json](https://github.com/kubernetes/node-problem-detector/blob/master/config/kernel-monitor.json).
   Node problem detector will start a separate log monitor for each configuration. You can
   use different log monitors to monitor different system log.
-* `--config.custom-plugin-monitor`: List of paths to custom plugin monitor config files, comma separated, e.g.
-  [config/custom-plugin-monitor.json](https://github.com/kubernetes/node-problem-detector/blob/master/config/custom-plugin-monitor.json).
-  Node problem detector will start a separate custom plugin monitor for each configuration. You can
-  use different custom plugin monitors to monitor different node problems.
+
+#### For System Stats Monitor
+
 * `--config.system-stats-monitor`: List of paths to system stats monitor config files, comma separated, e.g.
   [config/system-stats-monitor.json](https://github.com/kubernetes/node-problem-detector/blob/master/config/system-stats-monitor.json).
   Node problem detector will start a separate system stats monitor for each configuration. You can
   use different system stats monitors to monitor different problem-related system stats.
+
+#### For Custom Plugin Monitor
+
+* `--config.custom-plugin-monitor`: List of paths to custom plugin monitor config files, comma separated, e.g.
+  [config/custom-plugin-monitor.json](https://github.com/kubernetes/node-problem-detector/blob/master/config/custom-plugin-monitor.json).
+  Node problem detector will start a separate custom plugin monitor for each configuration. You can
+  use different custom plugin monitors to monitor different node problems.
+
+#### For Kubernetes exporter
+
 * `--enable-k8s-exporter`: Enables reporting to Kubernetes API server, default to `true`.
 * `--apiserver-override`: A URI parameter used to customize how node-problem-detector
 connects the apiserver.  This is ignored if `--enable-k8s-exporter` is `false`. The format is same as the
@@ -100,10 +117,18 @@ For example, to run without auth, use the following config:
    http://APISERVER_IP:APISERVER_PORT?inClusterConfig=false
    ```
    Refer [heapster docs](https://github.com/kubernetes/heapster/blob/master/docs/source-configuration.md#kubernetes) for a complete list of available options.
-* `--hostname-override`: A customized node name used for node-problem-detector to update conditions and emit events. node-problem-detector gets node name first from `hostname-override`, then `NODE_NAME` environment variable and finally fall back to `os.Hostname`.
+* `--address`: The address to bind the node problem detector server.
+* `--port`: The port to bind the node problem detector server. Use 0 to disable.
+
+#### For Prometheus exporter
+
 * `--prometheus-address`: The address to bind the Prometheus scrape endpoint, default to `127.0.0.1`.
 * `--prometheus-port`: The port to bind the Prometheus scrape endpoint, default to 20257. Use 0 to disable.
 
+#### For Stackdriver exporter
+
+* `--exporter.stackdriver`: Path to a Stackdriver exporter config file, e.g. [config/exporter/stackdriver-exporter.json](https://github.com/kubernetes/node-problem-detector/blob/master/config/exporter/stackdriver-exporter.json), default to empty string. Set to empty string to disable.
+
 ### Deprecated Flags
 
 * `--system-log-monitors`: List of paths to system log monitor config files, comma separated. This option is deprecated, replaced by `--config.system-log-monitor`, and will be removed. NPD will panic if both `--system-log-monitors` and `--config.system-log-monitor` are set.

cmd/nodeproblemdetector/exporterplugins/default_plugin.go

@@ -0,0 +1,20 @@
+/*
+Copyright 2019 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package exporterplugins
+
+// This file is necessary to make sure the exporterplugins package non-empty
+// under any build tags.

cmd/nodeproblemdetector/exporterplugins/stackdriver_exporter_plugin.go

@@ -0,0 +1,25 @@
+// +build !disable_stackdriver_exporter
+
+/*
+Copyright 2019 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package exporterplugins
+
+import (
+	_ "k8s.io/node-problem-detector/pkg/exporters/stackdriver"
+)
+
+// The stackdriver plugin takes about 6MB in the NPD binary.

cmd/nodeproblemdetector/node_problem_detector.go

@@ -22,8 +22,10 @@ import (
 	"github.com/golang/glog"
 	"github.com/spf13/pflag"
 
+	_ "k8s.io/node-problem-detector/cmd/nodeproblemdetector/exporterplugins"
 	_ "k8s.io/node-problem-detector/cmd/nodeproblemdetector/problemdaemonplugins"
 	"k8s.io/node-problem-detector/cmd/options"
+	"k8s.io/node-problem-detector/pkg/exporters"
 	"k8s.io/node-problem-detector/pkg/exporters/k8sexporter"
 	"k8s.io/node-problem-detector/pkg/exporters/prometheusexporter"
 	"k8s.io/node-problem-detector/pkg/problemdaemon"
@@ -54,21 +56,28 @@ func main() {
 	}
 
 	// Initialize exporters.
-	exporters := []types.Exporter{}
+	defaultExporters := []types.Exporter{}
 	if ke := k8sexporter.NewExporterOrDie(npdo); ke != nil {
-		exporters = append(exporters, ke)
+		defaultExporters = append(defaultExporters, ke)
 		glog.Info("K8s exporter started.")
 	}
 	if pe := prometheusexporter.NewExporterOrDie(npdo); pe != nil {
-		exporters = append(exporters, pe)
+		defaultExporters = append(defaultExporters, pe)
 		glog.Info("Prometheus exporter started.")
 	}
-	if len(exporters) == 0 {
+
+	plugableExporters := exporters.NewExporters()
+
+	npdExporters := []types.Exporter{}
+	npdExporters = append(npdExporters, defaultExporters...)
+	npdExporters = append(npdExporters, plugableExporters...)
+
+	if len(npdExporters) == 0 {
 		glog.Fatalf("No exporter is successfully setup")
 	}
 
 	// Initialize NPD core.
-	p := problemdetector.NewProblemDetector(problemDaemons, exporters)
+	p := problemdetector.NewProblemDetector(problemDaemons, npdExporters)
 	if err := p.Run(); err != nil {
 		glog.Fatalf("Problem detector failed with error: %v", err)
 	}

cmd/options/options.go

@@ -26,6 +26,7 @@ import (
 
 	"github.com/spf13/pflag"
 
+	"k8s.io/node-problem-detector/pkg/exporters"
 	"k8s.io/node-problem-detector/pkg/problemdaemon"
 	"k8s.io/node-problem-detector/pkg/types"
 )
@@ -86,6 +87,7 @@ type NodeProblemDetectorOptions struct {
 
 func NewNodeProblemDetectorOptions() *NodeProblemDetectorOptions {
 	npdo := &NodeProblemDetectorOptions{MonitorConfigPaths: types.ProblemDaemonConfigPathMap{}}
+
 	for _, problemDaemonName := range problemdaemon.GetProblemDaemonNames() {
 		npdo.MonitorConfigPaths[problemDaemonName] = &[]string{}
 	}
@@ -118,6 +120,10 @@ func (npdo *NodeProblemDetectorOptions) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&npdo.PrometheusServerAddress, "prometheus-address",
 		"127.0.0.1", "The address to bind the Prometheus scrape endpoint.")
 
+	for _, exporterName := range exporters.GetExporterNames() {
+		exporterHandler := exporters.GetExporterHandlerOrDie(exporterName)
+		exporterHandler.Options.SetFlags(fs)
+	}
 	for _, problemDaemonName := range problemdaemon.GetProblemDaemonNames() {
 		fs.StringSliceVar(
 			npdo.MonitorConfigPaths[problemDaemonName],

config/exporter/stackdriver-exporter.json

@@ -0,0 +1,8 @@
+{
+	"apiEndpoint": "monitoring.googleapis.com:443",
+	"exportPeriod": "60s",
+	"metadataFetchTimeout": "600s",
+	"metadataFetchInterval": "10s",
+	"panicOnMetadataFetchFailure": false,
+	"customMetricPrefix": ""
+}

config/systemd/node-problem-detector-metric-only.service

@@ -1,12 +1,13 @@
 [Unit]
 Description=Node problem detector
-Wants=local-fs.target
-After=local-fs.target
+Wants=network-online.target
+After=network-online.target
 
 [Service]
 Restart=always
 RestartSec=10
 ExecStart=/home/kubernetes/bin/node-problem-detector --v=2 --logtostderr --enable-k8s-exporter=false \
+          --exporter.stackdriver=/home/kubernetes/node-problem-detector/config/exporter/stackdriver-exporter.json \
           --config.system-log-monitor=/home/kubernetes/node-problem-detector/config/kernel-monitor.json,/home/kubernetes/node-problem-detector/config/docker-monitor.json,/home/kubernetes/node-problem-detector/config/systemd-monitor.json \
           --config.custom-plugin-monitor=/home/kubernetes/node-problem-detector/config/kernel-monitor-counter.json,/home/kubernetes/node-problem-detector/config/systemd-monitor-counter.json \
           --config.system-stats-monitor=/home/kubernetes/node-problem-detector/config/system-stats-monitor.json          

pkg/exporters/register.go

@@ -0,0 +1,63 @@
+/*
+Copyright 2019 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package exporters
+
+import (
+	"fmt"
+
+	"k8s.io/node-problem-detector/pkg/types"
+)
+
+var (
+	handlers = make(map[types.ExporterType]types.ExporterHandler)
+)
+
+// Register registers a exporter factory method, which will be used to create the exporter.
+func Register(exporterType types.ExporterType, handler types.ExporterHandler) {
+	handlers[exporterType] = handler
+}
+
+// GetExporterNames retrieves all available exporter types.
+func GetExporterNames() []types.ExporterType {
+	exporterTypes := []types.ExporterType{}
+	for exporterType := range handlers {
+		exporterTypes = append(exporterTypes, exporterType)
+	}
+	return exporterTypes
+}
+
+// GetExporterHandlerOrDie retrieves the ExporterHandler for a specific type of exporter, panic if error occurs..
+func GetExporterHandlerOrDie(exporterType types.ExporterType) types.ExporterHandler {
+	handler, ok := handlers[exporterType]
+	if !ok {
+		panic(fmt.Sprintf("Exporter handler for %v does not exist", exporterType))
+	}
+	return handler
+}
+
+// NewExporters creates all exporters based on the configurations initialized.
+func NewExporters() []types.Exporter {
+	exporters := []types.Exporter{}
+	for _, handler := range handlers {
+		exporter := handler.CreateExporterOrDie(handler.Options)
+		if exporter == nil {
+			continue
+		}
+		exporters = append(exporters, exporter)
+	}
+	return exporters
+}