[DNM] downburst sshtunnel

docs/siteconfig.rst

@@ -255,6 +255,17 @@ Here is a sample configuration with many of the options set and documented::
       endpoint: http://head.ses.suse.de:5000/
       machine_types: ['type1', 'type2', 'type3']
 
+    # Define a list of ssh tunnels for a various group of test nodes.
+    # Notice: provided domain names for the nodes must be resolvable
+    # in your network and jump host (bastion) must be accessible.
+    tunnel:
+      - hosts: ['example1.domain', 'example2.domain', 'example3.domain']
+        bastion:
+          host: ssh_host_name         # must be resolvable and reachable
+          user: ssh_user_name         # (optional)
+          port: ssh_port              # (optional)
+          identity: ~/.ssh/id_ed25519 # (optional)
+
     # Do not allow more than that many jobs in a single run by default.
     # To disable this check use 0.
     job_threshold: 500

setup.cfg

@@ -56,6 +56,7 @@ install_requires =
     python-dateutil
     requests>2.13.0
     sentry-sdk
+    sshtunnel
     types-psutil
     urllib3>=1.25.4,<1.27  # For botocore
 scripts =

teuthology/misc.py

@@ -1065,6 +1065,11 @@ def _ssh_keyscan(hostname):
     :returns: The host key
     """
     args = ['ssh-keyscan', '-T', '1', hostname]
+    if config.tunnel:
+        for tunnel in config.tunnel:
+            if hostname in tunnel.get('hosts'):
+                bastion = tunnel.get('bastion')
+                args = ['ssh', bastion.get('host')] + args
     p = subprocess.Popen(
         args=args,
         stdout=subprocess.PIPE,

teuthology/orchestra/connection.py

@@ -8,6 +8,7 @@
 from teuthology.config import config
 from teuthology.contextutil import safe_while
 from paramiko.hostkeys import HostKeyEntry
+import sshtunnel
 
 log = logging.getLogger(__name__)
 
@@ -50,7 +51,20 @@ def connect(user_at_host, host_key=None, keep_alive=False, timeout=60,
     :param retry:       Whether or not to retry failed connection attempts
                         (eventually giving up if none succeed). Default is True
     :param key_filename:  Optionally override which private key to use.
+
     :return: ssh connection.
+
+    The connection is going to be established via tunnel if corresponding options
+    are provided in teuthology configuration file. For example:
+
+    tunnel:
+      - hosts: ['hostname1.domain', 'hostname2.domain']
+        bastion:
+          host: ssh_host_name
+          user: ssh_user_name
+          port: 22
+          identity: ~/.ssh/id_ed25519
+
     """
     user, host = split_user(user_at_host)
     if _SSHClient is None:
@@ -79,6 +93,29 @@ def connect(user_at_host, host_key=None, keep_alive=False, timeout=60,
         timeout=timeout
     )
 
+    if config.tunnel:
+        for tunnel in config.tunnel:
+            if host in tunnel.get('hosts'):
+                bastion = tunnel.get('bastion')
+                if not bastion:
+                    log.error("The 'tunnel' config must include 'bastion' entry")
+                    continue
+                bastion_host = bastion.get('host')
+                server = sshtunnel.SSHTunnelForwarder(
+                    bastion_host,
+                    ssh_username=bastion.get('user', None),
+                    ssh_password=bastion.get('word', None),
+                    ssh_pkey=bastion.get('identity'),
+                    remote_bind_address=(host, 22))
+                log.info(f'Starting tunnel to {bastion_host} for host {host}')
+                server.start()
+                local_port = server.local_bind_port
+                log.debug(f"Local port for host {host} is {local_port}")
+                connect_args['hostname'] = '127.0.0.1'
+                connect_args['port'] = local_port
+                ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+                break
+
     key_filename = key_filename or config.ssh_key
     ssh_config_path = config.ssh_config_path or "~/.ssh/config"
     ssh_config_path = os.path.expanduser(ssh_config_path)

teuthology/provision/downburst.py

@@ -265,6 +265,11 @@ def build_config(self):
         # to install 'python' to get python2.7, which ansible needs
         if os_type in ('ubuntu', 'fedora'):
             user_info['packages'].append('python')
+        if os_type in ('centos'):
+            user_info['packages'].extend([
+                'python3-pip',
+                'bind-utils',
+            ])
         user_fd = tempfile.NamedTemporaryFile(delete=False, mode='wt')
         user_str = "#cloud-config\n" + yaml.safe_dump(user_info)
         user_fd.write(user_str)

teuthology/task/ansible.py

@@ -259,13 +259,61 @@ def generate_inventory(self):
         we're using an existing file.
         """
         hosts = self.cluster.remotes.keys()
-        hostnames = [remote.hostname for remote in hosts]
-        hostnames.sort()
+        hostnames = []
+        proxy = []
+        for remote in hosts:
+            if teuth_config.tunnel:
+                for tunnel in teuth_config.tunnel:
+                    cmd = None
+                    if remote.hostname in tunnel.get('hosts'):
+                        bastion = tunnel.get('bastion')
+                        if not bastion:
+                            log.error("The 'tunnel' config must include 'bastion' entry")
+                            continue
+                        host = bastion.get('host', None)
+                        if not host:
+                            log.error("Bastion host is not provided. Tunnel ignored.")
+                            continue
+                        user = bastion.get('user', None)
+                        word = bastion.get('word', None)
+                        port = bastion.get('port', 22)
+                        pkey = bastion.get('identity', None)
+                        opts = "-W %h:%p"
+                        if word:
+                            log.warning(f"Password authentication requested for the bastion '{host}' "
+                                        f"in order to connect to remote '{remote.hostname}'. "
+                                        f"The password authentication is not supported and will be ignored")
+                        if port:
+                            opts += f" -p {port}"
+                        if pkey:
+                            opts += f" -i {pkey}"
+                        if user:
+                            opts += f" {user}@{host}"
+                        else:
+                            opts += f" {host}"
+                        cmd = f"ssh {opts}"
+                        if not host in proxy:
+                            proxy.append(host)
+                        break
+                if cmd:
+                    i = f"{remote.hostname} ansible_ssh_common_args='-o ProxyCommand=\"{cmd}\" -o StrictHostKeyChecking=no'"
+                else:
+                    i = remote.hostname
+            else:
+                i = remote.hostname
+            hostnames.append(i)
         inventory = []
         if self.inventory_group:
             inventory.append('[{0}]'.format(self.inventory_group))
-        inventory.extend(hostnames + [''])
+
+        inventory.extend(sorted(hostnames) + [''])
+
+        if len(proxy) > 0:
+            inventory.append('[proxy]')
+            inventory.extend(sorted(proxy) + [''])
+
         hosts_str = '\n'.join(inventory)
+
         self.inventory = self._write_inventory_files(hosts_str)
         self.generated_inventory = True
 

teuthology/task/pexec.py

@@ -51,7 +51,7 @@ def _exec_host(barrier, barrier_queue, remote, sudo, testdir, ls):
         if l == "barrier":
             _do_barrier(barrier, barrier_queue, remote)
             continue
-
+        log.debug(f"{remote.name}< {l}")
         r.stdin.writelines([l, '\n'])
         r.stdin.flush()
     r.stdin.writelines(['\n'])