Add user known hosts file support for SSH connections (#4085)

Main use case is e2e tests - running them locally will lead to changes
in ~/.ssh/known_hosts file. Which later will lead to failures, when our
clenaup jobs invalidate secrete scopes with cluster ssh keys.

Author: ilia-db

acceptance/ssh/connection/known_hosts

@@ -0,0 +1,2 @@
+# not actually checked by tests
+cluster ssh-rsa key

acceptance/ssh/connection/script

@@ -1,4 +1,4 @@
-errcode $CLI ssh connect --cluster=$TEST_DEFAULT_CLUSTER_ID --releases-dir=$CLI_RELEASES_DIR -- "echo 'Connection successful'" >out.stdout.txt 2>LOG.stderr
+errcode $CLI ssh connect --cluster=$TEST_DEFAULT_CLUSTER_ID --releases-dir=$CLI_RELEASES_DIR --user-known-hosts-file=known_hosts -- "echo 'Connection successful'" >out.stdout.txt 2>LOG.stderr
 
 if ! grep -q "Connection successful" out.stdout.txt; then
   run_id=$(cat LOG.stderr | grep -o "Job submitted successfully with run ID: [0-9]*" | grep -o "[0-9]*$")

experimental/ssh/cmd/connect.go

@@ -29,6 +29,7 @@ the SSH server and handling the connection proxy.
 	var handoverTimeout time.Duration
 	var releasesDir string
 	var autoStartCluster bool
+	var userKnownHostsFile string
 
 	cmd.Flags().StringVar(&clusterID, "cluster", "", "Databricks cluster ID (required)")
 	cmd.MarkFlagRequired("cluster")
@@ -46,6 +47,9 @@ the SSH server and handling the connection proxy.
 	cmd.Flags().StringVar(&releasesDir, "releases-dir", "", "Directory for local SSH tunnel development releases")
 	cmd.Flags().MarkHidden("releases-dir")
 
+	cmd.Flags().StringVar(&userKnownHostsFile, "user-known-hosts-file", "", "Path to user known hosts file for SSH client")
+	cmd.Flags().MarkHidden("user-known-hosts-file")
+
 	cmd.PreRunE = func(cmd *cobra.Command, args []string) error {
 		// CLI in the proxy mode is executed by the ssh client and can't prompt for input
 		if proxyMode {
@@ -73,6 +77,7 @@ the SSH server and handling the connection proxy.
 			AutoStartCluster:     autoStartCluster,
 			ClientPublicKeyName:  clientPublicKeyName,
 			ClientPrivateKeyName: clientPrivateKeyName,
+			UserKnownHostsFile:   userKnownHostsFile,
 			AdditionalArgs:       args,
 		}
 		return client.Run(ctx, wsClient, opts)

experimental/ssh/internal/client/client.go

@@ -68,6 +68,8 @@ type ClientOptions struct {
 	Profile string
 	// Additional arguments to pass to the SSH client in the non proxy mode.
 	AdditionalArgs []string
+	// Optional path to the user known hosts file.
+	UserKnownHostsFile string
 }
 
 func Run(ctx context.Context, client *databricks.WorkspaceClient, opts ClientOptions) error {
@@ -253,8 +255,11 @@ func spawnSSHClient(ctx context.Context, userName, privateKeyPath string, server
 		"-o", "StrictHostKeyChecking=accept-new",
 		"-o", "ConnectTimeout=360",
 		"-o", "ProxyCommand=" + proxyCommand,
-		opts.ClusterID,
 	}
+	if opts.UserKnownHostsFile != "" {
+		sshArgs = append(sshArgs, "-o", "UserKnownHostsFile="+opts.UserKnownHostsFile)
+	}
+	sshArgs = append(sshArgs, opts.ClusterID)
 	sshArgs = append(sshArgs, opts.AdditionalArgs...)
 
 	cmdio.LogString(ctx, "Launching SSH client: ssh "+strings.Join(sshArgs, " "))