fix red herring in the readinessProbe (#1504)

* fix red herring

* refactor logging

Author: nammn

cmd/readiness/main.go

@@ -221,12 +221,22 @@ func main() {
 		panic(err)
 	}
 
-	cfg, err := config.BuildFromEnvVariables(clientSet, isHeadlessMode())
+	initLogger(config.GetLogger())
+
+	healthStatusFilePath := config.GetEnvOrDefault(config.AgentHealthStatusFilePathEnv, config.DefaultAgentHealthStatusFilePath)
+	file, err := os.Open(healthStatusFilePath)
+	// The agent might be slow in creating the health status file.
+	// In that case, we don't want to panic to show the message
+	// in the kubernetes description. That would be a red herring, since that will solve itself with enough time.
 	if err != nil {
-		panic(err)
+		logger.Errorf("health status file not avaible yet: %s ", err)
+		os.Exit(1)
 	}
 
-	initLogger(cfg.Logger)
+	cfg, err := config.BuildFromEnvVariables(clientSet, isHeadlessMode(), file)
+	if err != nil {
+		panic(err)
+	}
 
 	ready, err := isPodReady(cfg)
 	if err != nil {

pkg/readiness/config/config.go

@@ -13,16 +13,17 @@ import (
 )
 
 const (
-	defaultAgentHealthStatusFilePath = "/var/log/mongodb-mms-automation/agent-health-status.json"
-	defaultLogPath                   = "/var/log/mongodb-mms-automation/readiness.log"
-	podNamespaceEnv                  = "POD_NAMESPACE"
-	automationConfigSecretEnv        = "AUTOMATION_CONFIG_MAP" //nolint
-	agentHealthStatusFilePathEnv     = "AGENT_STATUS_FILEPATH"
-	logPathEnv                       = "LOG_FILE_PATH"
-	hostNameEnv                      = "HOSTNAME"
-	readinessProbeLoggerBackups      = "READINESS_PROBE_LOGGER_BACKUPS"
-	readinessProbeLoggerMaxSize      = "READINESS_PROBE_LOGGER_MAX_SIZE"
-	readinessProbeLoggerMaxAge       = "READINESS_PROBE_LOGGER_MAX_AGE"
+	DefaultAgentHealthStatusFilePath = "/var/log/mongodb-mms-automation/agent-health-status.json"
+	AgentHealthStatusFilePathEnv     = "AGENT_STATUS_FILEPATH"
+
+	defaultLogPath              = "/var/log/mongodb-mms-automation/readiness.log"
+	podNamespaceEnv             = "POD_NAMESPACE"
+	automationConfigSecretEnv   = "AUTOMATION_CONFIG_MAP" //nolint
+	logPathEnv                  = "LOG_FILE_PATH"
+	hostNameEnv                 = "HOSTNAME"
+	readinessProbeLoggerBackups = "READINESS_PROBE_LOGGER_BACKUPS"
+	readinessProbeLoggerMaxSize = "READINESS_PROBE_LOGGER_MAX_SIZE"
+	readinessProbeLoggerMaxAge  = "READINESS_PROBE_LOGGER_MAX_AGE"
 )
 
 type Config struct {
@@ -32,12 +33,10 @@ type Config struct {
 	AutomationConfigSecretName string
 	HealthStatusReader         io.Reader
 	LogFilePath                string
-	Logger                     *lumberjack.Logger
 }
 
-func BuildFromEnvVariables(clientSet kubernetes.Interface, isHeadless bool) (Config, error) {
-	healthStatusFilePath := getEnvOrDefault(agentHealthStatusFilePathEnv, defaultAgentHealthStatusFilePath)
-	logFilePath := getEnvOrDefault(logPathEnv, defaultLogPath)
+func BuildFromEnvVariables(clientSet kubernetes.Interface, isHeadless bool, file *os.File) (Config, error) {
+	logFilePath := GetEnvOrDefault(logPathEnv, defaultLogPath)
 
 	var namespace, automationConfigName, hostname string
 	if isHeadless {
@@ -56,35 +55,33 @@ func BuildFromEnvVariables(clientSet kubernetes.Interface, isHeadless bool) (Con
 		}
 	}
 
-	logger := &lumberjack.Logger{
-		Filename:   readinessProbeLogFilePath(),
-		MaxBackups: readIntOrDefault(readinessProbeLoggerBackups, 5),
-		MaxSize:    readInt(readinessProbeLoggerMaxSize),
-		MaxAge:     readInt(readinessProbeLoggerMaxAge),
-	}
-
 	// Note, that we shouldn't close the file here - it will be closed very soon by the 'ioutil.ReadAll'
 	// in main.go
-	file, err := os.Open(healthStatusFilePath)
-	if err != nil {
-		return Config{}, err
-	}
 	return Config{
 		ClientSet:                  clientSet,
 		Namespace:                  namespace,
 		AutomationConfigSecretName: automationConfigName,
 		Hostname:                   hostname,
 		HealthStatusReader:         file,
 		LogFilePath:                logFilePath,
-		Logger:                     logger,
 	}, nil
 }
 
+func GetLogger() *lumberjack.Logger {
+	logger := &lumberjack.Logger{
+		Filename:   readinessProbeLogFilePath(),
+		MaxBackups: readIntOrDefault(readinessProbeLoggerBackups, 5),
+		MaxSize:    readInt(readinessProbeLoggerMaxSize),
+		MaxAge:     readInt(readinessProbeLoggerMaxAge),
+	}
+	return logger
+}
+
 func readinessProbeLogFilePath() string {
-	return getEnvOrDefault(logPathEnv, defaultLogPath)
+	return GetEnvOrDefault(logPathEnv, defaultLogPath)
 }
 
-func getEnvOrDefault(envVar, defaultValue string) string {
+func GetEnvOrDefault(envVar, defaultValue string) string {
 	value := strings.TrimSpace(os.Getenv(envVar))
 	if value == "" {
 		return defaultValue
@@ -101,7 +98,7 @@ func readInt(envVarName string) int {
 // readIntOrDefault returns the int value of an envvar of the given name.
 // defaults to the given value if not specified.
 func readIntOrDefault(envVarName string, defaultValue int) int {
-	envVar := getEnvOrDefault(envVarName, strconv.Itoa(defaultValue))
+	envVar := GetEnvOrDefault(envVarName, strconv.Itoa(defaultValue))
 	intValue, err := strconv.Atoi(envVar)
 	if err != nil {
 		return defaultValue

release.json

@@ -2,7 +2,7 @@
   "golang-builder-image": "golang:1.21",
   "operator": "0.9.0",
   "version-upgrade-hook": "1.0.8",
-  "readiness-probe": "1.0.17",
+  "readiness-probe": "1.0.18",
   "agent": "107.0.1.8507-1",
   "agent-tools-version": "100.9.4"
 }