upgrade primary

Author: Ben Iofel

go.mod

@@ -8,8 +8,10 @@ require (
 	github.com/hashicorp/consul/api v1.18.0
 	github.com/jackc/pgconn v1.14.3
 	github.com/jackc/pgx/v5 v5.5.4
+	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
 	github.com/pkg/term v1.1.0
+	github.com/spf13/cobra v1.8.1
 	github.com/superfly/fly-checks v0.0.0-20230510154016-d189351293f2
 	golang.org/x/exp v0.0.0-20230105202349-8879d0199aa3
 	golang.org/x/sync v0.1.0
@@ -36,8 +38,6 @@ require (
 	github.com/mattn/go-runewidth v0.0.9 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/mapstructure v1.4.1 // indirect
-	github.com/olekukonko/tablewriter v0.0.5 // indirect
-	github.com/spf13/cobra v1.8.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/objx v0.5.0 // indirect
 	golang.org/x/crypto v0.20.0 // indirect

internal/flypg/readonly.go

@@ -70,7 +70,7 @@ func BroadcastReadonlyChange(ctx context.Context, n *Node, enabled bool) error {
 
 	for _, member := range members {
 		if member.Role == PrimaryRoleName {
-			endpoint := fmt.Sprintf("http://%s:5500/%s", fmt.Sprintf("%s.vm.%s.internal", member.NodeName, n.AppName), target)
+			endpoint := fmt.Sprintf("http://%s:5500/%s", member.NodeName, target)
 			resp, err := http.Get(endpoint)
 			if err != nil {
 				log.Printf("[WARN] Failed to broadcast readonly state change to member %s: %s", member.NodeName, err)
@@ -85,7 +85,7 @@ func BroadcastReadonlyChange(ctx context.Context, n *Node, enabled bool) error {
 	}
 
 	for _, member := range members {
-		endpoint := fmt.Sprintf("http://%s:5500/%s", fmt.Sprintf("%s.vm.%s.internal", member.NodeName, n.AppName), RestartHaproxyEndpoint)
+		endpoint := fmt.Sprintf("http://%s:5500/%s", member.NodeName, RestartHaproxyEndpoint)
 		resp, err := http.Get(endpoint)
 		if err != nil {
 			log.Printf("[WARN] Failed to restart haproxy on member %s: %s", member.NodeName, err)

internal/flypg/repmgr.go

@@ -174,11 +174,11 @@ func (r *RepMgr) setDefaults() error {
 		return err
 	}
 
-	hostname := fmt.Sprintf("%s.vm.%s.internal", r.MachineID, r.AppName)
+	hostname := r.machineIdToDNS(r.MachineID)
 
 	conf := ConfigMap{
 		"node_id":                      nodeID,
-		"node_name":                    fmt.Sprintf("'%s'", r.MachineID),
+		"node_name":                    fmt.Sprintf("'%s'", hostname),
 		"conninfo":                     fmt.Sprintf("'host=%s port=%d user=%s dbname=%s connect_timeout=5'", hostname, r.Port, r.Credentials.Username, r.DatabaseName),
 		"data_directory":               fmt.Sprintf("'%s'", r.DataDir),
 		"failover":                     "'automatic'",
@@ -301,10 +301,10 @@ func (r *RepMgr) unregisterWitness(id int) error {
 	return err
 }
 
-func (r *RepMgr) rejoinCluster(machineID string) error {
+func (r *RepMgr) rejoinCluster(nodeName string) error {
 	cmdStr := fmt.Sprintf("repmgr -f %s node rejoin -h %s -p %d -U %s -d %s --force-rewind --no-wait",
 		r.ConfigPath,
-		fmt.Sprintf("%s.vm.%s.internal", machineID, r.AppName),
+		nodeName,
 		r.Port,
 		r.Credentials.Username,
 		r.DatabaseName,
@@ -316,14 +316,14 @@ func (r *RepMgr) rejoinCluster(machineID string) error {
 	return err
 }
 
-func (r *RepMgr) clonePrimary(machineId string) error {
+func (r *RepMgr) clonePrimary(hostname string) error {
 	cmdStr := fmt.Sprintf("mkdir -p %s", r.DataDir)
 	if _, err := utils.RunCommand(cmdStr, "postgres"); err != nil {
 		return fmt.Errorf("failed to create pg directory: %s", err)
 	}
 
 	cmdStr = fmt.Sprintf("repmgr -h %s -p %d -d %s -U %s -f %s standby clone -c -F",
-		fmt.Sprintf("%s.vm.%s.internal", machineId, r.AppName),
+		hostname,
 		r.Port,
 		r.DatabaseName,
 		r.Credentials.Username,
@@ -474,13 +474,13 @@ func (r *RepMgr) ResolveMemberOverDNS(ctx context.Context) (*Member, error) {
 			continue
 		}
 
-		conn, err := r.NewRemoteConnection(ctx, machineId)
+		conn, err := r.NewRemoteConnection(ctx, r.machineIdToDNS(machineId))
 		if err != nil {
 			continue
 		}
 		defer func() { _ = conn.Close(ctx) }()
 
-		member, err := r.MemberByHostname(ctx, conn, machineId)
+		member, err := r.MemberByHostname(ctx, conn, r.machineIdToDNS(machineId))
 		if err != nil {
 			continue
 		}
@@ -559,3 +559,11 @@ func (r *RepMgr) UnregisterMember(member Member) error {
 func (r *RepMgr) eligiblePrimary() bool {
 	return r.Region == r.PrimaryRegion
 }
+
+func (r *RepMgr) machineIdToDNS(nodeName string) string {
+	if len(nodeName) != 14 {
+		panic("invalid machine id")
+	}
+
+	return fmt.Sprintf("%s.vm.%s.internal", nodeName, r.AppName)
+}

internal/flypg/zombie.go

@@ -117,7 +117,7 @@ func TakeDNASample(ctx context.Context, node *Node, standbys []Member) (*DNASamp
 		sample.totalActive++
 
 		// Record conflict when primary doesn't match.
-		if primary.NodeName != node.MachineID {
+		if primary.NodeName != node.MachineID && primary.NodeName != node.PrivateIP {
 			sample.totalConflicts++
 			sample.conflictMap[primary.NodeName]++
 		}