Bug fix: Resolve member by id rather than by ip (#186)

* Resolve member by id rather than by ip

* Reregister ip

* Bug fix

* Fix lint

Author: davissp14

bin/restart-repmgrd

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+kill `cat /tmp/repmgrd.pid`

internal/flypg/node.go

@@ -260,6 +260,10 @@ func (n *Node) PostInit(ctx context.Context) error {
 			return fmt.Errorf("failed to resolve member role: %s", err)
 		}
 
+		// Restart repmgrd in the event the IP changes for an already registered node.
+		// This can happen if the underlying volume is moved to a different node.
+		daemonRestartRequired := n.RepMgr.daemonRestartRequired(member)
+
 		switch member.Role {
 		case PrimaryRoleName:
 			// Verify cluster state to ensure we are the actual primary and not a zombie.
@@ -290,15 +294,20 @@ func (n *Node) PostInit(ctx context.Context) error {
 				)
 			}
 
+			// Re-register primary to apply any configuration changes.
+			if err := n.RepMgr.registerPrimary(daemonRestartRequired); err != nil {
+				return fmt.Errorf("failed to re-register existing primary: %s", err)
+			}
+
 			// Readonly lock is set when disk capacity is dangerously high.
 			if !ReadOnlyLockExists() {
 				if err := BroadcastReadonlyChange(ctx, n, false); err != nil {
 					return fmt.Errorf("failed to unset read-only: %s", err)
 				}
 			}
 		case StandbyRoleName:
-			// Register existing standby to take-on any configuration changes.
-			if err := n.RepMgr.registerStandby(); err != nil {
+			// Register existing standby to apply any configuration changes.
+			if err := n.RepMgr.registerStandby(daemonRestartRequired); err != nil {
 				return fmt.Errorf("failed to register existing standby: %s", err)
 			}
 		case WitnessRoleName:
@@ -307,7 +316,7 @@ func (n *Node) PostInit(ctx context.Context) error {
 				return fmt.Errorf("failed to resolve primary member when updating witness: %s", err)
 			}
 
-			// Register existing witness to take-on any configuration changes.
+			// Register existing witness to apply any configuration changes.
 			if err := n.RepMgr.registerWitness(primary.Hostname); err != nil {
 				return fmt.Errorf("failed to register existing witness: %s", err)
 			}
@@ -357,7 +366,7 @@ func (n *Node) PostInit(ctx context.Context) error {
 			}
 
 			// Register ourself as the primary
-			if err := n.RepMgr.registerPrimary(); err != nil {
+			if err := n.RepMgr.registerPrimary(false); err != nil {
 				return fmt.Errorf("failed to register repmgr primary: %s", err)
 			}
 
@@ -395,7 +404,7 @@ func (n *Node) PostInit(ctx context.Context) error {
 				}
 			} else {
 				log.Println("Registering standby")
-				if err := n.RepMgr.registerStandby(); err != nil {
+				if err := n.RepMgr.registerStandby(false); err != nil {
 					return fmt.Errorf("failed to register new standby: %s", err)
 				}
 			}

internal/flypg/repmgr.go

@@ -217,18 +217,34 @@ func (r *RepMgr) resolveNodeID() (string, error) {
 	return nodeID, nil
 }
 
-func (r *RepMgr) registerPrimary() error {
+func (r *RepMgr) registerPrimary(restartDaemon bool) error {
 	cmdStr := fmt.Sprintf("repmgr primary register -f %s -F", r.ConfigPath)
-	_, err := utils.RunCommand(cmdStr, "postgres")
+	if _, err := utils.RunCommand(cmdStr, "postgres"); err != nil {
+		return fmt.Errorf("failed to register primary: %s", err)
+	}
 
-	return err
+	if restartDaemon {
+		if err := r.restartDaemon(); err != nil {
+			return fmt.Errorf("failed to restart repmgr daemon: %s", err)
+		}
+	}
+
+	return nil
 }
 
-func (r *RepMgr) registerStandby() error {
+func (r *RepMgr) registerStandby(restartDaemon bool) error {
 	cmdStr := fmt.Sprintf("repmgr standby register -f %s -F", r.ConfigPath)
-	_, err := utils.RunCommand(cmdStr, "postgres")
+	if _, err := utils.RunCommand(cmdStr, "postgres"); err != nil {
+		return fmt.Errorf("failed to register standby: %s", err)
+	}
 
-	return err
+	if restartDaemon {
+		if err := r.restartDaemon(); err != nil {
+			return fmt.Errorf("failed to restart repmgr daemon: %s", err)
+		}
+	}
+
+	return nil
 }
 
 func (r *RepMgr) registerWitness(primaryHostname string) error {
@@ -252,6 +268,15 @@ func (r *RepMgr) unregisterStandby(id int) error {
 	return err
 }
 
+func (*RepMgr) restartDaemon() error {
+	_, err := utils.RunCommand("restart-repmgrd", "postgres")
+	return err
+}
+
+func (r *RepMgr) daemonRestartRequired(m *Member) bool {
+	return m.Hostname != r.PrivateIP
+}
+
 func (r *RepMgr) unregisterWitness(id int) error {
 	cmdStr := fmt.Sprintf("repmgr witness unregister -f %s --node-id=%d", r.ConfigPath, id)
 	_, err := utils.RunCommand(cmdStr, "postgres")
@@ -325,13 +350,18 @@ func (*RepMgr) Members(ctx context.Context, pg *pgx.Conn) ([]Member, error) {
 }
 
 func (r *RepMgr) Member(ctx context.Context, conn *pgx.Conn) (*Member, error) {
+	myID, err := r.resolveNodeID()
+	if err != nil {
+		return nil, fmt.Errorf("failed to resolve node id: %s", err)
+	}
+
 	members, err := r.Members(ctx, conn)
 	if err != nil {
 		return nil, err
 	}
 
 	for _, member := range members {
-		if member.Hostname == r.PrivateIP {
+		if fmt.Sprint(member.ID) == myID {
 			return &member, nil
 		}
 	}