Merge pull request openshift#6919 from bfournie/agent-tui-ssh-check

OCPBUGS-8094: In agent 'wait-for bootstrap' command, test ssh to Node0

Author: openshift-merge-robot

pkg/agent/cluster.go

@@ -2,8 +2,10 @@ package agent
 
 import (
 	"context"
+	"net"
 	"os"
 	"path/filepath"
+	"strconv"
 	"time"
 
 	"github.com/go-openapi/strfmt"
@@ -12,6 +14,7 @@ import (
 
 	"github.com/openshift/assisted-service/client/installer"
 	"github.com/openshift/assisted-service/models"
+	"github.com/openshift/installer/pkg/gather/ssh"
 )
 
 // Cluster is a struct designed to help interact with the cluster that is
@@ -56,6 +59,7 @@ type clusterInstallStatusHistory struct {
 	ClusterInstallComplete                              bool
 	NotReadyTime                                        time.Time
 	ValidationResults                                   *validationResults
+	ClusterInitTime                                     time.Time
 }
 
 // NewCluster initializes a Cluster object
@@ -94,6 +98,7 @@ func NewCluster(ctx context.Context, assetDir string) (*Cluster, error) {
 		ClusterConsoleRouteCreated:    false,
 		ClusterConsoleRouteURLCreated: false,
 		ClusterInstallComplete:        false,
+		ClusterInitTime:               time.Now(),
 	}
 
 	cvalidationresults := &validationResults{
@@ -136,7 +141,14 @@ func (czero *Cluster) IsBootstrapComplete() (bool, bool, error) {
 		logrus.Trace("Current API Status: Agent Rest API: down, Bootstrap Kube API: down")
 		if !czero.installHistory.RestAPISeen && !czero.installHistory.ClusterKubeAPISeen {
 			logrus.Debug("Agent Rest API never initialized. Bootstrap Kube API never initialized")
-			logrus.Info("Waiting for cluster install to initialize. Sleeping for 30 seconds")
+			elapsedSinceInit := time.Since(czero.installHistory.ClusterInitTime)
+			// After allowing time for the interface to come up, check if Node0 can be accessed via ssh
+			if elapsedSinceInit > 2*time.Minute && !czero.CanSSHToNodeZero() {
+				logrus.Info("Cannot access Rendezvous Host. There may be a network configuration problem, check console for additional info")
+			} else {
+				logrus.Info("Waiting for cluster install to initialize. Sleeping for 30 seconds")
+			}
+
 			time.Sleep(30 * time.Second)
 			return false, false, nil
 		}
@@ -425,6 +437,18 @@ func (czero *Cluster) PrintInstallStatus(cluster *models.Cluster) error {
 	return nil
 }
 
+// CanSSHToNodeZero Checks if ssh to NodeZero succeeds.
+func (czero *Cluster) CanSSHToNodeZero() bool {
+	ip := czero.API.Rest.NodeZeroIP
+	port := 22
+
+	_, err := ssh.NewClient("core", net.JoinHostPort(ip, strconv.Itoa(port)), czero.API.Rest.NodeSSHKey)
+	if err != nil {
+		logrus.Debugf("Failed to connect to the Rendezvous Host: %s", err)
+	}
+	return err == nil
+}
+
 // Human friendly install status strings mapped to the Agent Rest API cluster statuses
 func humanFriendlyClusterInstallStatus(status string) string {
 	clusterStoppedInstallingStates := map[string]string{

pkg/agent/rest.go

@@ -17,6 +17,7 @@ import (
 	"github.com/openshift/installer/pkg/asset/agent/agentconfig"
 	"github.com/openshift/installer/pkg/asset/agent/image"
 	"github.com/openshift/installer/pkg/asset/agent/manifests"
+	"github.com/openshift/installer/pkg/asset/installconfig"
 	assetstore "github.com/openshift/installer/pkg/asset/store"
 	"github.com/openshift/installer/pkg/types/agent"
 )
@@ -27,13 +28,15 @@ type NodeZeroRestClient struct {
 	ctx        context.Context
 	config     client.Config
 	NodeZeroIP string
+	NodeSSHKey []string
 }
 
 // NewNodeZeroRestClient Initialize a new rest client to interact with the Agent Rest API on node zero.
 func NewNodeZeroRestClient(ctx context.Context, assetDir string) (*NodeZeroRestClient, error) {
 	restClient := &NodeZeroRestClient{}
 	agentConfigAsset := &agentconfig.AgentConfig{}
 	agentManifestsAsset := &manifests.AgentManifests{}
+	installConfigAsset := &installconfig.InstallConfig{}
 
 	assetStore, err := assetstore.NewStore(assetDir)
 	if err != nil {
@@ -42,15 +45,19 @@ func NewNodeZeroRestClient(ctx context.Context, assetDir string) (*NodeZeroRestC
 
 	agentConfig, agentConfigError := assetStore.Load(agentConfigAsset)
 	agentManifests, manifestError := assetStore.Load(agentManifestsAsset)
+	installConfig, installConfigError := assetStore.Load(installConfigAsset)
 
 	if agentConfigError != nil {
 		logrus.Debug(errors.Wrapf(agentConfigError, "failed to load %s", agentConfigAsset.Name()))
 	}
 	if manifestError != nil {
 		logrus.Debug(errors.Wrapf(manifestError, "failed to load %s", agentManifestsAsset.Name()))
 	}
-	if agentConfigError != nil || manifestError != nil {
-		return nil, errors.New("failed to load AgentConfig or NMStateConfig")
+	if installConfigError != nil {
+		logrus.Debug(errors.Wrapf(installConfigError, "failed to load %s", installConfigAsset.Name()))
+	}
+	if agentConfigError != nil || manifestError != nil || installConfigError != nil {
+		return nil, errors.New("failed to load AgentConfig, NMStateConfig, or InstallConfig")
 	}
 
 	var RendezvousIP string
@@ -70,6 +77,11 @@ func NewNodeZeroRestClient(ctx context.Context, assetDir string) (*NodeZeroRestC
 		return nil, rendezvousIPError
 	}
 
+	// Get SSH Keys which can be used to determine if Rest API failures are due to network connectivity issues
+	if installConfig != nil {
+		restClient.NodeSSHKey = append(restClient.NodeSSHKey, installConfig.(*installconfig.InstallConfig).Config.SSHKey)
+	}
+
 	config := client.Config{}
 	config.URL = &url.URL{
 		Scheme: "http",