Check NGINX worker processes to check if NGINX reload is complete (#1207)

Author: aphralG

api/grpc/mpi/v1/command.pb.go

@@ -379,28 +379,6 @@ func registerFlags() {
 		DefManifestDir,
 		"Specifies the path to the directory containing the manifest files",
 	)
-	fs.Duration(
-		NginxReloadMonitoringPeriodKey,
-		DefNginxReloadMonitoringPeriod,
-		"The amount of time to monitor NGINX after a reload of configuration.",
-	)
-	fs.Bool(
-		NginxTreatWarningsAsErrorsKey,
-		DefTreatErrorsAsWarnings,
-		"Warning messages in the NGINX errors logs after a NGINX reload will be treated as an error.",
-	)
-
-	fs.String(
-		NginxApiTlsCa,
-		DefNginxApiTlsCa,
-		"The NGINX Plus CA certificate file location needed to call the NGINX Plus API if SSL is enabled.",
-	)
-
-	fs.StringSlice(
-		NginxExcludeLogsKey, []string{},
-		"A comma-separated list of one or more NGINX log paths that you want to exclude from metrics "+
-			"collection or error monitoring. This includes absolute paths or regex patterns",
-	)
 
 	fs.StringSlice(AllowedDirectoriesKey,
 		DefaultAllowedDirectories(),
@@ -442,6 +420,7 @@ func registerFlags() {
 	registerAuxiliaryCommandFlags(fs)
 	registerCollectorFlags(fs)
 	registerClientFlags(fs)
+	registerDataPlaneFlags(fs)
 
 	fs.SetNormalizeFunc(normalizeFunc)
 
@@ -456,6 +435,57 @@ func registerFlags() {
 	})
 }
 
+func registerDataPlaneFlags(fs *flag.FlagSet) {
+	fs.Duration(
+		NginxReloadMonitoringPeriodKey,
+		DefNginxReloadMonitoringPeriod,
+		"The amount of time to monitor NGINX after a reload of configuration.",
+	)
+	fs.Bool(
+		NginxTreatWarningsAsErrorsKey,
+		DefTreatErrorsAsWarnings,
+		"Warning messages in the NGINX errors logs after a NGINX reload will be treated as an error.",
+	)
+
+	fs.String(
+		NginxApiTlsCa,
+		DefNginxApiTlsCa,
+		"The NGINX Plus CA certificate file location needed to call the NGINX Plus API if SSL is enabled.",
+	)
+
+	fs.StringSlice(
+		NginxExcludeLogsKey, []string{},
+		"A comma-separated list of one or more NGINX log paths that you want to exclude from metrics "+
+			"collection or error monitoring. This includes absolute paths or regex patterns",
+	)
+
+	// NGINX Reload Backoff Flags
+	fs.Duration(
+		NginxReloadBackoffInitialIntervalKey,
+		DefNginxReloadBackoffInitialInterval,
+		"The NGINX reload backoff initial interval, value in seconds")
+
+	fs.Duration(
+		NginxReloadBackoffMaxIntervalKey,
+		DefNginxReloadBackoffMaxInterval,
+		"The NGINX reload backoff max interval, value in seconds")
+
+	fs.Duration(
+		NginxReloadBackoffMaxElapsedTimeKey,
+		DefNginxReloadBackoffMaxElapsedTime,
+		"The NGINX reload backoff max elapsed time, value in seconds")
+
+	fs.Float64(
+		NginxReloadBackoffRandomizationFactorKey,
+		DefNginxReloadBackoffRandomizationFactor,
+		"The NGINX reload backoff randomization factor, value float")
+
+	fs.Float64(
+		NginxReloadBackoffMultiplierKey,
+		DefNginxReloadBackoffMultiplier,
+		"The NGINX reload backoff multiplier, value float")
+}
+
 func registerCommonFlags(fs *flag.FlagSet) {
 	fs.StringToString(
 		LabelsRootKey,
@@ -906,6 +936,13 @@ func resolveDataPlaneConfig() *DataPlaneConfig {
 			TreatWarningsAsErrors:  viperInstance.GetBool(NginxTreatWarningsAsErrorsKey),
 			ExcludeLogs:            viperInstance.GetStringSlice(NginxExcludeLogsKey),
 			APITls:                 TLSConfig{Ca: viperInstance.GetString(NginxApiTlsCa)},
+			ReloadBackoff: &BackOff{
+				InitialInterval:     viperInstance.GetDuration(NginxReloadBackoffInitialIntervalKey),
+				MaxInterval:         viperInstance.GetDuration(NginxReloadBackoffMaxIntervalKey),
+				MaxElapsedTime:      viperInstance.GetDuration(NginxReloadBackoffMaxElapsedTimeKey),
+				RandomizationFactor: viperInstance.GetFloat64(NginxReloadBackoffRandomizationFactorKey),
+				Multiplier:          viperInstance.GetFloat64(NginxReloadBackoffMultiplierKey),
+			},
 		},
 	}
 }

api/grpc/mpi/v1/common.pb.go

@@ -1101,6 +1101,13 @@ func createConfig() *Config {
 				ExcludeLogs:            []string{"/var/log/nginx/error.log", "^/var/log/nginx/.*.log$"},
 				ReloadMonitoringPeriod: 30 * time.Second,
 				TreatWarningsAsErrors:  true,
+				ReloadBackoff: &BackOff{
+					InitialInterval:     100 * time.Millisecond,
+					MaxInterval:         20 * time.Second,
+					MaxElapsedTime:      15 * time.Second,
+					RandomizationFactor: 1.5,
+					Multiplier:          1.5,
+				},
 			},
 		},
 		Collector: &Collector{

api/grpc/mpi/v1/files.pb.go

@@ -16,6 +16,13 @@ const (
 	DefTreatErrorsAsWarnings       = false
 	DefNginxApiTlsCa               = ""
 
+	// Nginx Reload Backoff defaults
+	DefNginxReloadBackoffInitialInterval     = 1 * time.Second
+	DefNginxReloadBackoffRandomizationFactor = 0.5 // the value is 0 <= and < 1
+	DefNginxReloadBackoffMultiplier          = 5
+	DefNginxReloadBackoffMaxInterval         = 10 * time.Second
+	DefNginxReloadBackoffMaxElapsedTime      = 30 * time.Second
+
 	DefCommandServerHostKey    = ""
 	DefCommandServerPortKey    = 0
 	DefCommandServerTypeKey    = "grpc"

internal/config/config.go

@@ -112,10 +112,16 @@ var (
 	LogLevelKey = pre(LogLevelRootKey) + "level"
 	LogPathKey  = pre(LogLevelRootKey) + "path"
 
-	NginxReloadMonitoringPeriodKey = pre(DataPlaneConfigRootKey, "nginx") + "reload_monitoring_period"
-	NginxTreatWarningsAsErrorsKey  = pre(DataPlaneConfigRootKey, "nginx") + "treat_warnings_as_errors"
-	NginxExcludeLogsKey            = pre(DataPlaneConfigRootKey, "nginx") + "exclude_logs"
-	NginxApiTlsCa                  = pre(DataPlaneConfigRootKey, "nginx") + "api_tls_ca"
+	NginxReloadMonitoringPeriodKey           = pre(DataPlaneConfigRootKey, "nginx") + "reload_monitoring_period"
+	NginxTreatWarningsAsErrorsKey            = pre(DataPlaneConfigRootKey, "nginx") + "treat_warnings_as_errors"
+	NginxReloadBackoffKey                    = pre(DataPlaneConfigRootKey, "nginx") + "reload_backoff"
+	NginxReloadBackoffInitialIntervalKey     = pre(NginxReloadBackoffKey) + "initial_interval"
+	NginxReloadBackoffMaxIntervalKey         = pre(NginxReloadBackoffKey) + "max_interval"
+	NginxReloadBackoffMaxElapsedTimeKey      = pre(NginxReloadBackoffKey) + "max_elapsed_time"
+	NginxReloadBackoffRandomizationFactorKey = pre(NginxReloadBackoffKey) + "randomization_factor"
+	NginxReloadBackoffMultiplierKey          = pre(NginxReloadBackoffKey) + "multiplier"
+	NginxExcludeLogsKey                      = pre(DataPlaneConfigRootKey, "nginx") + "exclude_logs"
+	NginxApiTlsCa                            = pre(DataPlaneConfigRootKey, "nginx") + "api_tls_ca"
 
 	FileWatcherMonitoringFrequencyKey = pre(FileWatcherKey) + "monitoring_frequency"
 	NginxExcludeFilesKey              = pre(FileWatcherKey) + "exclude_files"

internal/config/config_test.go

@@ -31,6 +31,12 @@ data_plane_config:
         exclude_logs: 
             - /var/log/nginx/error.log
             - ^/var/log/nginx/.*.log$
+        reload_backoff:
+            initial_interval: 100ms
+            max_interval: 20s
+            max_elapsed_time: 15s
+            randomization_factor: 1.5
+            multiplier: 1.5
 client:
     http: 
         timeout: 15s

internal/config/defaults.go

@@ -63,6 +63,7 @@ type (
 	}
 
 	NginxDataPlaneConfig struct {
+		ReloadBackoff          *BackOff      `yaml:"reload_backoff"           mapstructure:"reload_backoff"`
 		APITls                 TLSConfig     `yaml:"api_tls"                  mapstructure:"api_tls"`
 		ExcludeLogs            []string      `yaml:"exclude_logs"             mapstructure:"exclude_logs"`
 		ReloadMonitoringPeriod time.Duration `yaml:"reload_monitoring_period" mapstructure:"reload_monitoring_period"`

internal/config/flags.go

@@ -0,0 +1,158 @@
+// Copyright (c) F5, Inc.
+//
+// This source code is licensed under the Apache License, Version 2.0 license found in the
+// LICENSE file in the root directory of this source tree.
+
+package nginx
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"fmt"
+	"regexp"
+	"strings"
+
+	"github.com/nginx/agent/v3/internal/datasource/host/exec"
+	"github.com/nginx/agent/v3/internal/model"
+	"github.com/nginx/agent/v3/pkg/nginxprocess"
+)
+
+const (
+	keyValueLen = 2
+	flagLen     = 1
+)
+
+var versionRegex = regexp.MustCompile(`(?P<name>\S+)\/(?P<version>.*)`)
+
+func ProcessInfo(ctx context.Context, proc *nginxprocess.Process,
+	executer exec.ExecInterface,
+) (*model.ProcessInfo, error) {
+	exePath := proc.Exe
+
+	if exePath == "" {
+		exePath = Exe(ctx, executer)
+		if exePath == "" {
+			return nil, fmt.Errorf("unable to find NGINX exe for process %d", proc.PID)
+		}
+	}
+
+	confPath := ConfPathFromCommand(proc.Cmd)
+
+	var nginxInfo *model.ProcessInfo
+
+	outputBuffer, err := executer.RunCmd(ctx, exePath, "-V")
+	if err != nil {
+		return nil, err
+	}
+
+	nginxInfo = ParseNginxVersionCommandOutput(ctx, outputBuffer)
+
+	nginxInfo.ExePath = exePath
+	nginxInfo.ProcessID = proc.PID
+
+	if nginxInfo.ConfPath = model.NginxConfPath(ctx, nginxInfo); confPath != "" {
+		nginxInfo.ConfPath = confPath
+	}
+
+	return nginxInfo, err
+}
+
+func Exe(ctx context.Context, executer exec.ExecInterface) string {
+	exePath := ""
+
+	out, commandErr := executer.RunCmd(ctx, "sh", "-c", "command -v nginx")
+	if commandErr == nil {
+		exePath = strings.TrimSuffix(out.String(), "\n")
+	}
+
+	if exePath == "" {
+		exePath = defaultToNginxCommandForProcessPath(executer)
+	}
+
+	if strings.Contains(exePath, "(deleted)") {
+		exePath = sanitizeExeDeletedPath(exePath)
+	}
+
+	return exePath
+}
+
+func defaultToNginxCommandForProcessPath(executer exec.ExecInterface) string {
+	exePath, err := executer.FindExecutable("nginx")
+	if err != nil {
+		return ""
+	}
+
+	return exePath
+}
+
+func sanitizeExeDeletedPath(exe string) string {
+	firstSpace := strings.Index(exe, "(deleted)")
+	if firstSpace != -1 {
+		return strings.TrimSpace(exe[0:firstSpace])
+	}
+
+	return strings.TrimSpace(exe)
+}
+
+func ConfPathFromCommand(command string) string {
+	commands := strings.Split(command, " ")
+
+	for i, command := range commands {
+		if command == "-c" {
+			if i < len(commands)-1 {
+				return commands[i+1]
+			}
+		}
+	}
+
+	return ""
+}
+
+func ParseNginxVersionCommandOutput(ctx context.Context, output *bytes.Buffer) *model.ProcessInfo {
+	nginxInfo := &model.ProcessInfo{}
+
+	scanner := bufio.NewScanner(output)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		switch {
+		case strings.HasPrefix(line, "nginx version"):
+			nginxInfo.Version = parseNginxVersion(line)
+		case strings.HasPrefix(line, "configure arguments"):
+			nginxInfo.ConfigureArgs = parseConfigureArguments(line)
+		}
+	}
+
+	nginxInfo.Prefix = model.NginxPrefix(ctx, nginxInfo)
+
+	return nginxInfo
+}
+
+func parseNginxVersion(line string) string {
+	return strings.TrimPrefix(versionRegex.FindString(line), "nginx/")
+}
+
+func parseConfigureArguments(line string) map[string]interface{} {
+	// need to check for empty strings
+	flags := strings.Split(line[len("configure arguments:"):], " --")
+	result := make(map[string]interface{})
+
+	for _, flag := range flags {
+		vals := strings.Split(flag, "=")
+		if isFlag(vals) {
+			result[vals[0]] = true
+		} else if isKeyValueFlag(vals) {
+			result[vals[0]] = vals[1]
+		}
+	}
+
+	return result
+}
+
+func isFlag(vals []string) bool {
+	return len(vals) == flagLen && vals[0] != ""
+}
+
+func isKeyValueFlag(vals []string) bool {
+	return len(vals) == keyValueLen
+}