Adjusting how Zombie state is evaluated on boot.

Author: davissp14

internal/flypg/node.go

@@ -231,14 +231,6 @@ func (n *Node) Init(ctx context.Context) error {
 
 // PostInit are operations that need to be executed against a running Postgres on boot.
 func (n *Node) PostInit(ctx context.Context) error {
-	if ZombieLockExists() {
-		log.Println("[ERROR] Manual intervention required.")
-		log.Println("[ERROR] If a new primary has been established, consider adding a new replica with `fly machines clone <primary-machine-id>` and then remove this member.")
-		log.Println("[ERROR] Sleeping for 5 minutes.")
-		time.Sleep(5 * time.Minute)
-		return fmt.Errorf("unrecoverable zombie")
-	}
-
 	// Use the Postgres user on boot, since our internal user may not have been created yet.
 	conn, err := n.NewLocalConnection(ctx, "postgres", n.OperatorCredentials)
 	if err != nil {
@@ -297,14 +289,22 @@ func (n *Node) PostInit(ctx context.Context) error {
 				return fmt.Errorf("failed to run zombie diagnosis: %s", err)
 			}
 
-			// This should never happen
-			if primary != n.PrivateIP {
+			// This should never happen, but check anyways for correctness
+			if primary != n.Hostname() {
 				return fmt.Errorf("resolved primary '%s' does not match ourself '%s'. this should not happen",
 					primary,
-					n.PrivateIP,
+					n.Hostname(),
 				)
 			}
 
+			// Clear the zombie lock if it exists.
+			if ZombieLockExists() {
+				log.Println("[INFO] Clearing zombie lock and re-enabling read/write")
+				if err := RemoveZombieLock(); err != nil {
+					return fmt.Errorf("failed to remove zombie lock: %s", err)
+				}
+			}
+
 			// Re-register primary to apply any configuration changes.
 			if err := n.RepMgr.registerPrimary(daemonRestartRequired); err != nil {
 				return fmt.Errorf("failed to re-register existing primary: %s", err)

internal/flypg/zombie.go

@@ -117,7 +117,7 @@ func TakeDNASample(ctx context.Context, node *Node, standbys []Member) (*DNASamp
 		sample.totalActive++
 
 		// Record conflict when primary name does not match our machine ID
-		if primary.Hostname != node.Hostname() {
+		if primary.Hostname != node.Hostname() && primary.Hostname != node.PrivateIP {
 			sample.totalConflicts++
 			sample.conflictMap[primary.Hostname]++
 		}
@@ -244,6 +244,7 @@ func handleZombieLock(ctx context.Context, n *Node) error {
 		// TODO - Provide link to documentation on how to address this
 		log.Println("[WARN] Zombie lock file does not contain a hostname.")
 		log.Println("[WARN] This likely means that we were unable to determine who the real primary is.")
+		log.Println("[WARN] If a new primary has been established, consider adding a new replica with `fly machines clone <primary-machine-id>` and then remove this member.")
 	}
 
 	return nil