Retry connection if nginx unavailable

Closes: #36

* Add nginx.retries cli flag
* Add nginx.retry-interval cli flag
* Add handling of SIGTERM signals
* Add started log message
* Add createClientWithRetries test
* Add time.Duration env variables: timeout, nginxRetryInterval
* Fix error context: On OSS Failed to create client would be appended twice from nginx client and exporter
* Add environment variables to --help flag and readme.md

Author: Dean-Coakley

README.md

@@ -64,13 +64,17 @@ To start the exporter we use the [docker run](https://docs.docker.com/engine/ref
 Usage of ./nginx-prometheus-exporter:
   -nginx.plus
         Start the exporter for NGINX Plus. By default, the exporter is started for NGINX. The default value can be overwritten by NGINX_PLUS environment variable.
+  -nginx.retries int
+        A number of retries the exporter will make on start to connect to the NGINX stub_status page/NGINX Plus API before exiting with an error. The default value can be overwritten by NGINX_RETRIES environment variable.
+  -nginx.retry-interval duration
+        An interval between retries to connect to the NGINX stub_status page/NGINX Plus API on start. The default value can be overwritten by NGINX_RETRY_INTERVAL environment variable. (default 5s)
   -nginx.scrape-uri string
         A URI for scraping NGINX or NGINX Plus metrics.
         For NGINX, the stub_status page must be available through the URI. For NGINX Plus -- the API. The default value can be overwritten by SCRAPE_URI environment variable. (default "http://127.0.0.1:8080/stub_status")
   -nginx.ssl-verify
         Perform SSL certificate verification. The default value can be overwritten by SSL_VERIFY environment variable. (default true)
   -nginx.timeout duration
-        A timeout for scraping metrics from NGINX or NGINX Plus. (default 5s)
+        A timeout for scraping metrics from NGINX or NGINX Plus. The default value can be overwritten by TIMEOUT environment variable. (default 5s)
   -web.listen-address string
         An address to listen on for web interface and telemetry. The default value can be overwritten by LISTEN_ADDRESS environment variable. (default ":9113")
   -web.telemetry-path string
@@ -84,7 +88,7 @@ Usage of ./nginx-prometheus-exporter:
 * For NGINX, the following metrics are exported:
     * All [stub_status](http://nginx.org/en/docs/http/ngx_http_stub_status_module.html) metrics.
     * `nginx_up` -- shows the status of the last metric scrape: `1` for a successful scrape and `0` for a failed one.
-    
+
     Connect to the `/metrics` page of the running exporter to see the complete list of metrics along with their descriptions.
 * For NGINX Plus, the following metrics are exported:
     * [Connections](http://nginx.org/en/docs/http/ngx_http_api_module.html#def_nginx_connections).

client/nginx.go

@@ -37,11 +37,8 @@ func NewNginxClient(httpClient *http.Client, apiEndpoint string) (*NginxClient,
 		httpClient:  httpClient,
 	}
 
-	if _, err := client.GetStubStats(); err != nil {
-		return nil, fmt.Errorf("Failed to create NginxClient: %v", err)
-	}
-
-	return client, nil
+	_, err := client.GetStubStats()
+	return client, err
 }
 
 // GetStubStats fetches the stub_status metrics.

exporter.go

@@ -6,7 +6,9 @@ import (
 	"log"
 	"net/http"
 	"os"
+	"os/signal"
 	"strconv"
+	"syscall"
 	"time"
 
 	plusclient "github.com/nginxinc/nginx-plus-go-sdk/client"
@@ -24,43 +26,100 @@ func getEnv(key, defaultValue string) string {
 	return value
 }
 
+func getEnvInt(key string, defaultValue int) int {
+	value, ok := os.LookupEnv(key)
+	if !ok {
+		return defaultValue
+	}
+	i, err := strconv.ParseInt(value, 10, 64)
+	if err != nil {
+		log.Fatalf("Environment variable value for %s must be an int: %v", key, err)
+	}
+	return int(i)
+}
+
 func getEnvBool(key string, defaultValue bool) bool {
 	value, ok := os.LookupEnv(key)
 	if !ok {
 		return defaultValue
 	}
 	b, err := strconv.ParseBool(value)
 	if err != nil {
-		log.Fatalf("Environment Variable value for %s must be a boolean", key)
+		log.Fatalf("Environment variable value for %s must be a boolean: %v", key, err)
 	}
 	return b
 }
 
+func getEnvDuration(key string, defaultValue time.Duration) time.Duration {
+	value, ok := os.LookupEnv(key)
+	if !ok {
+		return defaultValue
+	}
+	d, err := time.ParseDuration(value)
+	if err != nil {
+		log.Fatalf("Environment variable value for %s must be a duration: %v", key, err)
+	}
+	return d
+}
+
+func createClientWithRetries(getClient func() (interface{}, error), retries int, retryInterval time.Duration) (interface{}, error) {
+	var err error
+	var nginxClient interface{}
+
+	for i := retries; i >= 0; i-- {
+		nginxClient, err = getClient()
+		if err == nil {
+			return nginxClient, nil
+		}
+		if i > 0 {
+			log.Printf("Could not create Nginx Client. Retrying in %v...", retryInterval)
+			time.Sleep(retryInterval)
+		}
+	}
+	return nil, err
+}
+
 var (
 	// Set during go build
 	version   string
 	gitCommit string
 
 	// Defaults values
-	defaultListenAddress = getEnv("LISTEN_ADDRESS", ":9113")
-	defaultMetricsPath   = getEnv("TELEMETRY_PATH", "/metrics")
-	defaultNginxPlus     = getEnvBool("NGINX_PLUS", false)
-	defaultScrapeURI     = getEnv("SCRAPE_URI", "http://127.0.0.1:8080/stub_status")
-	defaultSslVerify     = getEnvBool("SSL_VERIFY", true)
+	defaultListenAddress      = getEnv("LISTEN_ADDRESS", ":9113")
+	defaultMetricsPath        = getEnv("TELEMETRY_PATH", "/metrics")
+	defaultNginxPlus          = getEnvBool("NGINX_PLUS", false)
+	defaultScrapeURI          = getEnv("SCRAPE_URI", "http://127.0.0.1:8080/stub_status")
+	defaultSslVerify          = getEnvBool("SSL_VERIFY", true)
+	defaultTimeout            = getEnvDuration("TIMEOUT", time.Second*5)
+	defaultNginxRetries       = getEnvInt("NGINX_RETRIES", 0)
+	defaultNginxRetryInterval = getEnvDuration("NGINX_RETRY_INTERVAL", time.Second*5)
 
 	// Command-line flags
-	listenAddr = flag.String("web.listen-address", defaultListenAddress,
+	listenAddr = flag.String("web.listen-address",
+		defaultListenAddress,
 		"An address to listen on for web interface and telemetry. The default value can be overwritten by LISTEN_ADDRESS environment variable.")
-	metricsPath = flag.String("web.telemetry-path", defaultMetricsPath,
+	metricsPath = flag.String("web.telemetry-path",
+		defaultMetricsPath,
 		"A path under which to expose metrics. The default value can be overwritten by TELEMETRY_PATH environment variable.")
-	nginxPlus = flag.Bool("nginx.plus", defaultNginxPlus,
+	nginxPlus = flag.Bool("nginx.plus",
+		defaultNginxPlus,
 		"Start the exporter for NGINX Plus. By default, the exporter is started for NGINX. The default value can be overwritten by NGINX_PLUS environment variable.")
-	scrapeURI = flag.String("nginx.scrape-uri", defaultScrapeURI,
+	scrapeURI = flag.String("nginx.scrape-uri",
+		defaultScrapeURI,
 		`A URI for scraping NGINX or NGINX Plus metrics.
 	For NGINX, the stub_status page must be available through the URI. For NGINX Plus -- the API. The default value can be overwritten by SCRAPE_URI environment variable.`)
-	sslVerify = flag.Bool("nginx.ssl-verify", defaultSslVerify,
+	sslVerify = flag.Bool("nginx.ssl-verify",
+		defaultSslVerify,
 		"Perform SSL certificate verification. The default value can be overwritten by SSL_VERIFY environment variable.")
-	timeout = flag.Duration("nginx.timeout", 5*time.Second, "A timeout for scraping metrics from NGINX or NGINX Plus.")
+	timeout = flag.Duration("nginx.timeout",
+		defaultTimeout,
+		"A timeout for scraping metrics from NGINX or NGINX Plus. The default value can be overwritten by TIMEOUT environment variable.")
+	nginxRetries = flag.Int("nginx.retries",
+		defaultNginxRetries,
+		"A number of retries the exporter will make on start to connect to the NGINX stub_status page/NGINX Plus API before exiting with an error. The default value can be overwritten by NGINX_RETRIES environment variable.")
+	nginxRetryInterval = flag.Duration("nginx.retry-interval",
+		defaultNginxRetryInterval,
+		"An interval between retries to connect to the NGINX stub_status page/NGINX Plus API on start. The default value can be overwritten by NGINX_RETRY_INTERVAL environment variable.")
 )
 
 func main() {
@@ -91,22 +150,30 @@ func main() {
 		},
 	}
 
+	signalChan := make(chan os.Signal, 1)
+	signal.Notify(signalChan, syscall.SIGTERM)
+	go func() {
+		log.Printf("SIGTERM received: %v. Exiting...", <-signalChan)
+		os.Exit(0)
+	}()
+
 	if *nginxPlus {
-		client, err := plusclient.NewNginxClient(httpClient, *scrapeURI)
+		plusClient, err := createClientWithRetries(func() (interface{}, error) {
+			return plusclient.NewNginxClient(httpClient, *scrapeURI)
+		}, *nginxRetries, *nginxRetryInterval)
 		if err != nil {
 			log.Fatalf("Could not create Nginx Plus Client: %v", err)
 		}
-
-		registry.MustRegister(collector.NewNginxPlusCollector(client, "nginxplus"))
+		registry.MustRegister(collector.NewNginxPlusCollector(plusClient.(*plusclient.NginxClient), "nginxplus"))
 	} else {
-		client, err := client.NewNginxClient(httpClient, *scrapeURI)
+		ossClient, err := createClientWithRetries(func() (interface{}, error) {
+			return client.NewNginxClient(httpClient, *scrapeURI)
+		}, *nginxRetries, *nginxRetryInterval)
 		if err != nil {
 			log.Fatalf("Could not create Nginx Client: %v", err)
 		}
-
-		registry.MustRegister(collector.NewNginxCollector(client, "nginx"))
+		registry.MustRegister(collector.NewNginxCollector(ossClient.(*client.NginxClient), "nginx"))
 	}
-
 	http.Handle(*metricsPath, promhttp.HandlerFor(registry, promhttp.HandlerOpts{}))
 	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
 		w.Write([]byte(`<html>
@@ -117,5 +184,6 @@ func main() {
 			</body>
 			</html>`))
 	})
+	log.Printf("NGINX Prometheus Exporter has successfully started")
 	log.Fatal(http.ListenAndServe(*listenAddr, nil))
 }

exporter_test.go

@@ -0,0 +1,83 @@
+package main
+
+import (
+	"errors"
+	"reflect"
+	"testing"
+	"time"
+)
+
+func TestCreateClientWithRetries(t *testing.T) {
+	type args struct {
+		client        interface{}
+		err           error
+		retries       int
+		retryInterval time.Duration
+	}
+
+	tests := []struct {
+		name            string
+		args            args
+		expectedRetries int
+		want            interface{}
+		wantErr         bool
+	}{
+		{
+			name: "getClient returns a valid client",
+			args: args{
+				client: "client",
+				err:    nil,
+			},
+			expectedRetries: 0,
+			want:            "client",
+			wantErr:         false,
+		},
+		{
+			name: "getClient returns an error after no retries",
+			args: args{
+				client: nil,
+				err:    errors.New("error"),
+			},
+			expectedRetries: 0,
+			want:            nil,
+			wantErr:         true,
+		},
+		{
+			name: "getClient returns an error after retries",
+			args: args{
+				client:        nil,
+				err:           errors.New("error"),
+				retries:       3,
+				retryInterval: time.Millisecond * 1,
+			},
+			expectedRetries: 3,
+			want:            nil,
+			wantErr:         true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			invocations := 0
+			getClient := func() (interface{}, error) {
+				invocations++
+				return tt.args.client, tt.args.err
+			}
+
+			got, err := createClientWithRetries(getClient, tt.args.retries, tt.args.retryInterval)
+
+			actualRetries := invocations - 1
+
+			if actualRetries != tt.expectedRetries {
+				t.Errorf("createClientWithRetries() got %v retries, expected %v", actualRetries, tt.expectedRetries)
+				return
+			} else if (err != nil) != tt.wantErr {
+				t.Errorf("createClientWithRetries() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			} else if err != nil && tt.wantErr {
+				return
+			} else if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("createClientWithRetries() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}