fix(fencing): allow fencing when WAL disk is full (cloudnative-pg#10302)

Move the WAL disk space check from the instance manager startup path
into the PostgreSQL lifecycle loop, after the fencing check. This
ensures the controller-runtime manager always starts, allowing fencing
annotations to be processed even when WAL disk is full.

Signed-off-by: Marco Nenciarini <marco.nenciarini@enterprisedb.com>
Signed-off-by: Armando Ruocco <armando.ruocco@enterprisedb.com>
Co-authored-by: Armando Ruocco <armando.ruocco@enterprisedb.com>
(cherry picked from commit f6d03dd)
(cherry picked from commit 72998cf)

Author: 2 people

internal/cmd/manager/instance/run/cmd.go

@@ -68,10 +68,6 @@ import (
 var (
 	scheme = runtime.NewScheme()
 
-	// errNoFreeWALSpace is returned when there isn't enough disk space
-	// available to store at least two WAL files.
-	errNoFreeWALSpace = fmt.Errorf("no free disk space for WALs")
-
 	// errWALArchivePluginNotAvailable is returned when the configured
 	// WAL archiving plugin is not available or cannot be found.
 	errWALArchivePluginNotAvailable = fmt.Errorf("WAL archive plugin not available")
@@ -127,7 +123,7 @@ func NewCmd() *cobra.Command {
 				return runSubCommand(ctx, instance, pprofHTTPServer, skipNameValidation)
 			})
 
-			if errors.Is(err, errNoFreeWALSpace) {
+			if errors.Is(err, postgres.ErrNoFreeWALSpace) {
 				os.Exit(apiv1.MissingWALDiskSpaceExitCode)
 			}
 			if errors.Is(err, errWALArchivePluginNotAvailable) {
@@ -180,15 +176,6 @@ func runSubCommand(
 		"build", versions.Info,
 		"skipNameValidation", skipNameValidation)
 
-	contextLogger.Info("Checking for free disk space for WALs before starting PostgreSQL")
-	hasDiskSpaceForWals, err := instance.CheckHasDiskSpaceForWAL(ctx)
-	if err != nil {
-		contextLogger.Error(err, "Error while checking if there is enough disk space for WALs, skipping")
-	} else if !hasDiskSpaceForWals {
-		contextLogger.Info("Detected low-disk space condition, avoid starting the instance")
-		return errNoFreeWALSpace
-	}
-
 	mgr, err := ctrl.NewManager(config.GetConfigOrDie(), ctrl.Options{
 		Scheme: scheme,
 		Cache: cache.Options{
@@ -409,16 +396,24 @@ func runSubCommand(
 	contextLogger.Info("starting controller-runtime manager")
 	if err := mgr.Start(onlineUpgradeCtx); err != nil {
 		contextLogger.Error(err, "unable to run controller-runtime manager")
+		if errors.Is(err, postgres.ErrNoFreeWALSpace) {
+			return makeUnretryableError(postgres.ErrNoFreeWALSpace)
+		}
+		if hasSpace, checkErr := instance.CheckHasDiskSpaceForWAL(ctx); checkErr == nil && !hasSpace {
+			contextLogger.Warning("Detected low WAL disk space, but the manager error is not WAL-space related",
+				"originalError", err)
+			return makeUnretryableError(fmt.Errorf("%w: %w", postgres.ErrNoFreeWALSpace, err))
+		}
 		return makeUnretryableError(err)
 	}
 
 	contextLogger.Info("Checking for free disk space for WALs after PostgreSQL finished")
-	hasDiskSpaceForWals, err = instance.CheckHasDiskSpaceForWAL(ctx)
+	hasDiskSpaceForWals, err := instance.CheckHasDiskSpaceForWAL(ctx)
 	if err != nil {
 		contextLogger.Error(err, "Error while checking if there is enough disk space for WALs, skipping")
 	} else if !hasDiskSpaceForWals {
 		contextLogger.Info("Detected low-disk space condition")
-		return makeUnretryableError(errNoFreeWALSpace)
+		return makeUnretryableError(postgres.ErrNoFreeWALSpace)
 	}
 
 	if instance.Cluster != nil {

internal/cmd/manager/instance/run/lifecycle/run.go

@@ -90,6 +90,13 @@ func (i *PostgresLifecycle) runPostgresAndWait(ctx context.Context) <-chan error
 			return nil
 		}
 
+		if hasDiskSpace, err := i.instance.CheckHasDiskSpaceForWAL(postgresContext); err != nil {
+			contextLogger.Error(err, "Error checking WAL disk space, skipping")
+		} else if !hasDiskSpace {
+			contextLogger.Info("Not enough WAL disk space, avoid starting PostgreSQL")
+			return postgres.ErrNoFreeWALSpace
+		}
+
 		i.instance.LogPgControldata(postgresContext, "postmaster start up")
 		defer i.instance.LogPgControldata(postgresContext, "postmaster has exited")
 

pkg/management/postgres/instance.go

@@ -133,6 +133,10 @@ var (
 
 	// ErrNoConnectionEstablished postgres is alive, but rejecting connections
 	ErrNoConnectionEstablished = fmt.Errorf("could not establish connection")
+
+	// ErrNoFreeWALSpace is returned when there isn't enough disk space
+	// available to store at least two WAL files.
+	ErrNoFreeWALSpace = errors.New("no free disk space for WALs")
 )
 
 // Instance represent a PostgreSQL instance to be executed