Resolve cherry-pick conflicts for #19427 backport to release-23.0

Resolve merge conflicts from cherry-picking the analysis ordering
and semi-sync rollout PR onto release-23.0. Adapt the new
problem-matching system to the release-23.0 codebase by removing
IncapacitatedPrimary (not present on this branch) and keeping
fmt.Sprintf style consistent with existing code.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: timvaillancourt

go/test/endtoend/vtorc/general/vtorc_test.go

@@ -20,11 +20,8 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-<<<<<<< HEAD
-=======
 	"strconv"
 	"strings"
->>>>>>> e7888dfa83 (`vtorc`: support analysis ordering, improve semi-sync rollout (#19427))
 	"testing"
 	"time"
 

go/vt/vtorc/inst/analysis_dao.go

@@ -422,141 +422,13 @@ func GetDetectionAnalysis(keyspace string, shard string, hints *DetectionAnalysi
 			return nil
 		}
 		isInvalid := m.GetBool("is_invalid")
-<<<<<<< HEAD
-		switch {
-		case a.IsClusterPrimary && isInvalid:
-			a.Analysis = InvalidPrimary
-			a.Description = "VTOrc hasn't been able to reach the primary even once since restart/shutdown"
-		case isInvalid:
-			a.Analysis = InvalidReplica
-			a.Description = "VTOrc hasn't been able to reach the replica even once since restart/shutdown"
-		case a.IsClusterPrimary && !a.LastCheckValid && a.IsDiskStalled:
-			a.Analysis = PrimaryDiskStalled
-			a.Description = "Primary has a stalled disk"
-			ca.hasShardWideAction = true
-		case a.IsClusterPrimary && !a.LastCheckValid && a.CountReplicas == 0:
-			a.Analysis = DeadPrimaryWithoutReplicas
-			a.Description = "Primary cannot be reached by vtorc and has no replica"
-			ca.hasShardWideAction = true
-			//
-		case a.IsClusterPrimary && !a.LastCheckValid && a.CountValidReplicas == a.CountReplicas && a.CountValidReplicatingReplicas == 0:
-			a.Analysis = DeadPrimary
-			a.Description = "Primary cannot be reached by vtorc and none of its replicas is replicating"
-			ca.hasShardWideAction = true
-			//
-		case a.IsClusterPrimary && !a.LastCheckValid && a.CountReplicas > 0 && a.CountValidReplicas == 0 && a.CountValidReplicatingReplicas == 0:
-			a.Analysis = DeadPrimaryAndReplicas
-			a.Description = "Primary cannot be reached by vtorc and none of its replicas is replicating"
-			ca.hasShardWideAction = true
-			//
-		case a.IsClusterPrimary && !a.LastCheckValid && a.CountValidReplicas < a.CountReplicas && a.CountValidReplicas > 0 && a.CountValidReplicatingReplicas == 0:
-			a.Analysis = DeadPrimaryAndSomeReplicas
-			a.Description = "Primary cannot be reached by vtorc; some of its replicas are unreachable and none of its reachable replicas is replicating"
-			ca.hasShardWideAction = true
-			//
-		case a.IsClusterPrimary && !a.IsPrimary:
-			a.Analysis = PrimaryHasPrimary
-			a.Description = "Primary is replicating from somewhere else"
-			ca.hasShardWideAction = true
-			//
-		case a.IsClusterPrimary && a.IsReadOnly:
-			a.Analysis = PrimaryIsReadOnly
-			a.Description = "Primary is read-only"
-			//
-		case a.IsClusterPrimary && policy.SemiSyncAckers(ca.durability, tablet) != 0 && !a.SemiSyncPrimaryEnabled:
-			a.Analysis = PrimarySemiSyncMustBeSet
-			a.Description = "Primary semi-sync must be set"
-			//
-		case a.IsClusterPrimary && policy.SemiSyncAckers(ca.durability, tablet) == 0 && a.SemiSyncPrimaryEnabled:
-			a.Analysis = PrimarySemiSyncMustNotBeSet
-			a.Description = "Primary semi-sync must not be set"
-			//
-		case a.IsClusterPrimary && a.CurrentTabletType != topodatapb.TabletType_UNKNOWN && a.CurrentTabletType != topodatapb.TabletType_PRIMARY:
-			a.Analysis = PrimaryCurrentTypeMismatch
-			a.Description = "Primary tablet's current type is not PRIMARY"
-		case isStaleTopoPrimary(a, ca):
-			a.Analysis = StaleTopoPrimary
-			a.Description = "Primary tablet is stale, older than current primary"
-		case topo.IsReplicaType(a.TabletType) && a.ErrantGTID != "":
-			a.Analysis = ErrantGTIDDetected
-			a.Description = "Tablet has errant GTIDs"
-		case topo.IsReplicaType(a.TabletType) && ca.primaryAlias == "" && a.ShardPrimaryTermTimestamp.IsZero():
-			// ClusterHasNoPrimary should only be detected when the shard record doesn't have any primary term start time specified either.
-			a.Analysis = ClusterHasNoPrimary
-			a.Description = "Cluster has no primary"
-			ca.hasShardWideAction = true
-		case topo.IsReplicaType(a.TabletType) && ca.primaryAlias == "" && !a.ShardPrimaryTermTimestamp.IsZero():
-			// If there are no primary tablets, but the shard primary start time isn't empty, then we know
-			// the primary tablet was deleted.
-			a.Analysis = PrimaryTabletDeleted
-			a.Description = "Primary tablet has been deleted"
-			ca.hasShardWideAction = true
-		case a.IsPrimary && a.SemiSyncBlocked && a.CountSemiSyncReplicasEnabled >= a.SemiSyncPrimaryWaitForReplicaCount:
-			// The primary is reporting that semi-sync monitor is blocked on writes.
-			// There are enough replicas configured to send semi-sync ACKs such that the primary shouldn't be blocked.
-			// There is some network diruption in progress. We should run an ERS.
-			a.Analysis = PrimarySemiSyncBlocked
-			a.Description = "Writes seem to be blocked on semi-sync acks on the primary, even though sufficient replicas are configured to send ACKs"
-			ca.hasShardWideAction = true
-		case topo.IsReplicaType(a.TabletType) && !a.IsReadOnly:
-			a.Analysis = ReplicaIsWritable
-			a.Description = "Replica is writable"
-			//
-		case topo.IsReplicaType(a.TabletType) && a.IsPrimary:
-			a.Analysis = NotConnectedToPrimary
-			a.Description = "Not connected to the primary"
-			//
-		case topo.IsReplicaType(a.TabletType) && !a.IsPrimary && math.Round(a.HeartbeatInterval*2) != float64(a.ReplicaNetTimeout):
-			a.Analysis = ReplicaMisconfigured
-			a.Description = "Replica has been misconfigured"
-			//
-		case topo.IsReplicaType(a.TabletType) && !a.IsPrimary && ca.primaryAlias != "" && a.AnalyzedInstancePrimaryAlias != ca.primaryAlias:
-			a.Analysis = ConnectedToWrongPrimary
-			a.Description = "Connected to wrong primary"
-			//
-		case topo.IsReplicaType(a.TabletType) && !a.IsPrimary && a.ReplicationStopped:
-			a.Analysis = ReplicationStopped
-			a.Description = "Replication is stopped"
-			//
-		case topo.IsReplicaType(a.TabletType) && !a.IsPrimary && policy.IsReplicaSemiSync(ca.durability, primaryTablet, tablet) && !a.SemiSyncReplicaEnabled:
-			a.Analysis = ReplicaSemiSyncMustBeSet
-			a.Description = "Replica semi-sync must be set"
-			//
-		case topo.IsReplicaType(a.TabletType) && !a.IsPrimary && !policy.IsReplicaSemiSync(ca.durability, primaryTablet, tablet) && a.SemiSyncReplicaEnabled:
-			a.Analysis = ReplicaSemiSyncMustNotBeSet
-			a.Description = "Replica semi-sync must not be set"
-			//
-			// TODO(sougou): Events below here are either ignored or not possible.
-		case a.IsPrimary && !a.LastCheckValid && a.CountLaggingReplicas == a.CountReplicas && a.CountDelayedReplicas < a.CountReplicas && a.CountValidReplicatingReplicas > 0:
-			a.Analysis = UnreachablePrimaryWithLaggingReplicas
-			a.Description = "Primary cannot be reached by vtorc and all of its replicas are lagging"
-			//
-		case a.IsPrimary && !a.LastCheckValid && !a.LastCheckPartialSuccess && a.CountValidReplicas > 0 && a.CountValidReplicatingReplicas == a.CountValidReplicas:
-			// partial success is here to reduce noise
-			a.Analysis = UnreachablePrimary
-			a.Description = "Primary cannot be reached by vtorc but all of its replicas seem to be replicating; possibly a network/host issue"
-			//
-		case a.IsPrimary && !a.LastCheckValid && !a.LastCheckPartialSuccess && a.CountValidReplicas > 0 && a.CountValidReplicatingReplicas > 0 && a.CountValidReplicatingReplicas < a.CountValidReplicas:
-			// partial success is here to reduce noise
-			a.Analysis = UnreachablePrimaryWithBrokenReplicas
-			a.Description = "Primary cannot be reached by vtorc but it has (some, but not all) replicating replicas; possibly a network/host issue"
-			//
-		case a.IsPrimary && a.SemiSyncPrimaryEnabled && a.SemiSyncPrimaryStatus && a.SemiSyncPrimaryWaitForReplicaCount > 0 && a.SemiSyncPrimaryClients < a.SemiSyncPrimaryWaitForReplicaCount:
-			if isStaleBinlogCoordinates {
-				a.Analysis = LockedSemiSyncPrimary
-				a.Description = "Semi sync primary is locked since it doesn't get enough replica acknowledgements"
-			} else {
-				a.Analysis = LockedSemiSyncPrimaryHypothesis
-				a.Description = "Semi sync primary seems to be locked, more samplings needed to validate"
-=======
 		var matchedProblems []*DetectionAnalysisProblem
 		for _, problem := range detectionAnalysisProblems {
 			// When isInvalid is true, instance data is unreliable (never been reached).
 			// Only InvalidPrimary/InvalidReplica should match; postProcessAnalyses
 			// handles upgrading InvalidPrimary to DeadPrimary if needed.
 			if isInvalid && problem.Meta.Analysis != InvalidPrimary && problem.Meta.Analysis != InvalidReplica {
 				continue
->>>>>>> e7888dfa83 (`vtorc`: support analysis ordering, improve semi-sync rollout (#19427))
 			}
 			if problem.HasMatch(a, ca, primaryTablet, tablet, isInvalid, isStaleBinlogCoordinates) {
 				matchedProblems = append(matchedProblems, problem)

go/vt/vtorc/inst/analysis_problem.go

@@ -173,18 +173,6 @@ var detectionAnalysisProblems = []*DetectionAnalysisProblem{
 		},
 	},
 
-	// IncapacitatedPrimary
-	{
-		Meta: &DetectionAnalysisProblemMeta{
-			Analysis:    IncapacitatedPrimary,
-			Description: "Primary is consistently timing out on health checks and may be incapacitated",
-			Priority:    detectionAnalysisPriorityShardWideAction,
-		},
-		MatchFunc: func(a *DetectionAnalysis, ca *clusterAnalysis, primary, tablet *topodatapb.Tablet, isInvalid, isStaleBinlogCoordinates bool) bool {
-			return a.IsClusterPrimary && !a.LastCheckValid && a.PrimaryHealthUnhealthy
-		},
-	},
-
 	// PrimaryHasPrimary
 	{
 		Meta: &DetectionAnalysisProblemMeta{

go/vt/vtorc/logic/topology_recovery.go

@@ -21,18 +21,10 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-<<<<<<< HEAD
-	"math/rand/v2"
-=======
-	"log/slog"
 	"maps"
 	"math/rand/v2"
-	"net"
-	"net/http"
 	"slices"
-	"strconv"
 	"sync"
->>>>>>> e7888dfa83 (`vtorc`: support analysis ordering, improve semi-sync rollout (#19427))
 	"sync/atomic"
 	"time"
 
@@ -1055,13 +1047,7 @@ func CheckAndRecover() {
 	})
 	for _, key := range shardKeys {
 		go func() {
-<<<<<<< HEAD
-			if err := executeCheckAndRecoverFunction(analysisEntry); err != nil {
-				log.Error(err)
-			}
-=======
 			recoverShardAnalyses(analysisByShard[key], executeCheckAndRecoverFunction)
->>>>>>> e7888dfa83 (`vtorc`: support analysis ordering, improve semi-sync rollout (#19427))
 		}()
 	}
 }