roachprod: fix service registry for unregistered services

As of #125393, we no longer register all virtual clusters
with DNS. This was an optimization made to reduce unnecessary
DNS usage when there was only one tenant using the default
ports. Specifically, we no longer register shared process
tenants and system tenants using the default ports.

However, some of the existing logic still assumes that
registration happens for all virtual clusters. For example,
clusterSynced.Stop errors out if a shared process virtual
cluster is not registered, even though this is now the
expected behavior.

This change establishes that DiscoverService only attempts
to find services, while ServiceDescriptor will always
return the service descriptor using fallback logic if needed.

Author: DarrylWong

pkg/roachprod/install/cluster_synced.go

@@ -464,31 +464,24 @@ func (c *SyncedCluster) Stop(
 	// killProcesses indicates whether processed need to be stopped.
 	killProcesses := true
 
-	// Non system shared process virtual clusters don't get killed but are stopped via SQL.
-	// Figure out if the virtual cluster is one or not.
+	// For shared process secondary tenants, we just stop the service via SQL.
+	// Find out of this is a shared process secondary tenant.
 	if virtualClusterLabel != "" {
 		name, sqlInstance, err := VirtualClusterInfoFromLabel(virtualClusterLabel)
 		if err != nil {
 			return err
 		}
 
-		if name != SystemInterfaceName {
-			services, err := c.DiscoverServices(ctx, name, ServiceTypeSQL)
+		if !IsSystemInterface(name) {
+			isExternal, err := c.IsExternalService(ctx, name)
 			if err != nil {
 				return err
 			}
 
-			if len(services) == 0 {
-				return fmt.Errorf("no service for virtual cluster %q", virtualClusterName)
-			}
-
-			virtualClusterName = name
-			if services[0].ServiceMode == ServiceModeShared {
-				// For shared process virtual clusters, we just stop the service
-				// via SQL.
-				killProcesses = false
-			} else {
+			if isExternal {
 				virtualClusterDisplay = fmt.Sprintf(" virtual cluster %q, instance %d", virtualClusterName, sqlInstance)
+			} else {
+				killProcesses = false
 			}
 		}
 	}
@@ -2268,7 +2261,7 @@ func (c *SyncedCluster) pgurls(
 	}
 	m := make(map[Node]string, len(hosts))
 	for node, host := range hosts {
-		desc, err := c.DiscoverService(ctx, node, virtualClusterName, ServiceTypeSQL, sqlInstance)
+		desc, err := c.ServiceDescriptor(ctx, node, virtualClusterName, ServiceTypeSQL, sqlInstance)
 		if err != nil {
 			return nil, err
 		}
@@ -2303,24 +2296,19 @@ func (c *SyncedCluster) loadBalancerURL(
 	sqlInstance int,
 	auth PGAuthMode,
 ) (string, error) {
-	services, err := c.DiscoverServices(ctx, virtualClusterName, ServiceTypeSQL)
+	// Note that it's possible for our service to not be running on the entire roachprod
+	// cluster, e.g. one of the nodes is a workload node, or we have a separate process
+	// virtual cluster running on a subset of nodes. We must search for our service on
+	// the entire cluster.
+	descs, err := c.ServiceDescriptors(ctx, c.Nodes, virtualClusterName, ServiceTypeSQL, sqlInstance)
 	if err != nil {
 		return "", err
 	}
-	port := config.DefaultSQLPort
-	serviceMode := ServiceModeExternal
-	for _, service := range services {
-		if service.VirtualClusterName == virtualClusterName && service.Instance == sqlInstance {
-			serviceMode = service.ServiceMode
-			port = service.Port
-			break
-		}
-	}
-	address, err := c.FindLoadBalancer(l, port)
+	address, err := c.FindLoadBalancer(l, descs[0].Port)
 	if err != nil {
 		return "", err
 	}
-	loadBalancerURL := c.NodeURL(address.IP, address.Port, virtualClusterName, serviceMode, auth, "" /* database */)
+	loadBalancerURL := c.NodeURL(address.IP, address.Port, virtualClusterName, descs[0].ServiceMode, auth, "" /* database */)
 	return loadBalancerURL, nil
 }
 
@@ -2725,7 +2713,7 @@ func (c *SyncedCluster) Reset(l *logger.Logger) error {
 		return nil
 	}
 
-	nodes := c.TargetNodes()
+	nodes := c.Nodes
 	targetVMs := make(vm.List, len(nodes))
 	for idx, node := range nodes {
 		targetVMs[idx] = c.VMs[node-1]