Merge pull request kubernetes#89231 from andrewsykim/e2e-framework-ssh

e2e/framework: implement ssh exec internally

Author: k8s-ci-robot

pkg/ssh/ssh.go

@@ -14,6 +14,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+// WARNING: DO NOT add new use-caes to this package as it is deprecated and slated for deletion.
+
 package ssh
 
 import (

test/e2e/framework/.import-restrictions

@@ -199,7 +199,6 @@
 				"k8s.io/kubernetes/pkg/security/podsecuritypolicy/util",
 				"k8s.io/kubernetes/pkg/securitycontext",
 				"k8s.io/kubernetes/pkg/serviceaccount",
-				"k8s.io/kubernetes/pkg/ssh",
 				"k8s.io/kubernetes/pkg/util/async",
 				"k8s.io/kubernetes/pkg/util/bandwidth",
 				"k8s.io/kubernetes/pkg/util/config",

test/e2e/framework/ssh/BUILD

@@ -6,7 +6,6 @@ go_library(
     importpath = "k8s.io/kubernetes/test/e2e/framework/ssh",
     visibility = ["//visibility:public"],
     deps = [
-        "//pkg/ssh:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library",

test/e2e/framework/ssh/ssh.go

@@ -20,19 +20,21 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io/ioutil"
 	"net"
 	"os"
 	"path/filepath"
 	"time"
 
 	"github.com/onsi/gomega"
+
 	"golang.org/x/crypto/ssh"
+
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
-	sshutil "k8s.io/kubernetes/pkg/ssh"
 	e2elog "k8s.io/kubernetes/test/e2e/framework/log"
 	testutils "k8s.io/kubernetes/test/utils"
 )
@@ -54,7 +56,7 @@ const (
 func GetSigner(provider string) (ssh.Signer, error) {
 	// honor a consistent SSH key across all providers
 	if path := os.Getenv("KUBE_SSH_KEY_PATH"); len(path) > 0 {
-		return sshutil.MakePrivateKeySignerFromFile(path)
+		return makePrivateKeySignerFromFile(path)
 	}
 
 	// Select the key itself to use. When implementing more providers here,
@@ -93,7 +95,21 @@ func GetSigner(provider string) (ssh.Signer, error) {
 		keyfile = filepath.Join(keydir, keyfile)
 	}
 
-	return sshutil.MakePrivateKeySignerFromFile(keyfile)
+	return makePrivateKeySignerFromFile(keyfile)
+}
+
+func makePrivateKeySignerFromFile(key string) (ssh.Signer, error) {
+	buffer, err := ioutil.ReadFile(key)
+	if err != nil {
+		return nil, fmt.Errorf("error reading SSH key %s: '%v'", key, err)
+	}
+
+	signer, err := ssh.ParsePrivateKey(buffer)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing SSH key: '%v'", err)
+	}
+
+	return signer, err
 }
 
 // NodeSSHHosts returns SSH-able host names for all schedulable nodes - this
@@ -169,14 +185,68 @@ func SSH(cmd, host, provider string) (Result, error) {
 		return result, err
 	}
 
-	stdout, stderr, code, err := sshutil.RunSSHCommand(cmd, result.User, host, signer)
+	stdout, stderr, code, err := runSSHCommand(cmd, result.User, host, signer)
 	result.Stdout = stdout
 	result.Stderr = stderr
 	result.Code = code
 
 	return result, err
 }
 
+// runSSHCommandViaBastion returns the stdout, stderr, and exit code from running cmd on
+// host as specific user, along with any SSH-level error.
+func runSSHCommand(cmd, user, host string, signer ssh.Signer) (string, string, int, error) {
+	if user == "" {
+		user = os.Getenv("USER")
+	}
+	// Setup the config, dial the server, and open a session.
+	config := &ssh.ClientConfig{
+		User:            user,
+		Auth:            []ssh.AuthMethod{ssh.PublicKeys(signer)},
+		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+	}
+	client, err := ssh.Dial("tcp", host, config)
+	if err != nil {
+		err = wait.Poll(5*time.Second, 20*time.Second, func() (bool, error) {
+			fmt.Printf("error dialing %s@%s: '%v', retrying\n", user, host, err)
+			if client, err = ssh.Dial("tcp", host, config); err != nil {
+				return false, err
+			}
+			return true, nil
+		})
+	}
+	if err != nil {
+		return "", "", 0, fmt.Errorf("error getting SSH client to %s@%s: '%v'", user, host, err)
+	}
+	defer client.Close()
+	session, err := client.NewSession()
+	if err != nil {
+		return "", "", 0, fmt.Errorf("error creating session to %s@%s: '%v'", user, host, err)
+	}
+	defer session.Close()
+
+	// Run the command.
+	code := 0
+	var bout, berr bytes.Buffer
+	session.Stdout, session.Stderr = &bout, &berr
+	if err = session.Run(cmd); err != nil {
+		// Check whether the command failed to run or didn't complete.
+		if exiterr, ok := err.(*ssh.ExitError); ok {
+			// If we got an ExitError and the exit code is nonzero, we'll
+			// consider the SSH itself successful (just that the command run
+			// errored on the host).
+			if code = exiterr.ExitStatus(); code != 0 {
+				err = nil
+			}
+		} else {
+			// Some other kind of error happened (e.g. an IOError); consider the
+			// SSH unsuccessful.
+			err = fmt.Errorf("failed running `%s` on %s@%s: '%v'", cmd, user, host, err)
+		}
+	}
+	return bout.String(), berr.String(), code, err
+}
+
 // runSSHCommandViaBastion returns the stdout, stderr, and exit code from running cmd on
 // host as specific user, along with any SSH-level error. It uses an SSH proxy to connect
 // to bastion, then via that tunnel connects to the remote host. Similar to