Merge pull request #509 from minrk/test-ssh

Test SSH launchers

Author: minrk

.github/workflows/test.yml

@@ -18,13 +18,15 @@ jobs:
       fail-fast: false
       matrix:
         include:
+          - python: "3.9"
+            ssh: ssh
           - python: "3.6"
             tornado: "5.1.1"
           - python: "3.6"
           - python: "3.7"
             controller_ip: "*"
           - python: "3.8"
-            mpi: true
+            mpi: mpi
           - python: "3.9"
 
     steps:
@@ -42,6 +44,14 @@ jobs:
         run: |
           echo "IPP_CONTROLLER_IP=${{ matrix.controller_ip }}" >> $GITHUB_ENV
 
+      - name: Set up docker-compose for ssh launcher
+        if: ${{ matrix.ssh }}
+        run: |
+          export DOCKER_BUILDKIT=1
+          export COMPOSE_DOCKER_CLI_BUILD=1
+          cd ci/ssh
+          docker-compose up -d --build
+
       - name: Install Python (conda) ${{ matrix.python }}
         if: matrix.mpi
         run: |
@@ -76,8 +86,13 @@ jobs:
         run: |
           pytest -vx --color=yes --cov=ipyparallel ipyparallel/tests/test_cluster.py
 
+      - name: Run ssh tests
+        if: ${{ matrix.ssh }}
+        run: |
+          pytest -ra -v --maxfail=2 --color=yes --cov=ipyparallel ipyparallel/tests/test_ssh.py
+
       - name: Run tests
-        if: ${{ ! matrix.mpi }}
+        if: ${{ ! matrix.mpi && ! matrix.ssh }}
         # FIXME: --color=yes explicitly set because:
         #        https://github.com/actions/runner/issues/241
         run: |

.gitignore

@@ -23,3 +23,5 @@ __pycache__
 .coverage.*
 .idea
 htmlcov
+id_*sa
+.vscode

ci/ssh/Dockerfile

@@ -0,0 +1,37 @@
+# syntax = docker/dockerfile:1.2.1
+FROM ubuntu:20.04
+RUN --mount=type=cache,target=/var/cache/apt \
+    rm -f /etc/apt/apt.conf.d/docker-clean \
+ && apt-get update \
+ && apt-get -y install wget openssh-server
+
+ENV MAMBA_ROOT_PREFIX=/opt/conda
+ENV PATH=$MAMBA_ROOT_PREFIX/bin:$PATH
+RUN wget -qO- https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba \
+ && mv bin/micromamba /usr/local/bin/micromamba
+
+RUN --mount=type=cache,target=${MAMBA_ROOT_PREFIX}/pkgs \
+    micromamba install -y -p $MAMBA_ROOT_PREFIX -c conda-forge \
+        python=3.8 \
+        pip \
+        ipyparallel
+
+# generate a user with home directory and trusted ssh keypair
+RUN useradd -m -s /bin/bash -N ciuser
+USER ciuser
+RUN mkdir ~/.ssh \
+ && chmod 0700 ~/.ssh \
+ && ssh-keygen -q -t rsa -N '' -f /home/ciuser/.ssh/id_rsa \
+ && cat ~/.ssh/id_rsa.pub > ~/.ssh/authorized_keys \
+ && chmod 0600 ~/.ssh/*
+USER root
+
+
+ENV PIP_CACHE_DIR=/tmp/pip-cache
+COPY . /src/ipyparallel
+RUN --mount=type=cache,target=${PIP_CACHE_DIR} python3 -m pip install -e 'file:///src/ipyparallel#egg=ipyparallel[test]'
+
+# needed for sshd to start
+RUN mkdir /run/sshd
+# run sshd in the foreground
+CMD /usr/sbin/sshd -D -e

ci/ssh/docker-compose.yaml

@@ -0,0 +1,8 @@
+services:
+  sshd:
+    image: ipyparallel-sshd
+    build:
+      context: ../..
+      dockerfile: ci/ssh/Dockerfile
+    ports:
+      - "2222:22"

ci/ssh/ipcluster_config.py

@@ -0,0 +1,20 @@
+import os
+
+c.Cluster.controller_ip = '0.0.0.0'
+c.Cluster.engine_launcher_class = 'SSH'
+
+ssh_key = os.path.join(os.path.dirname(__file__), "id_rsa")
+c.Cluster.controller_ip = '0.0.0.0'
+c.Cluster.engine_launcher_class = 'SSH'
+c.SSHEngineSetLauncher.scp_args = c.SSHLauncher.ssh_args = [
+    "-o",
+    "UserKnownHostsFile=/dev/null",
+    "-o",
+    "StrictHostKeyChecking=no",
+    "-i",
+    ssh_key,
+]
+c.SSHEngineSetLauncher.engines = {"[email protected]:2222": 4}
+c.SSHEngineSetLauncher.remote_python = "/opt/conda/bin/python3"
+c.SSHEngineSetLauncher.remote_profile_dir = "/home/ciuser/.ipython/profile_default"
+c.SSHEngineSetLauncher.engine_args = ['--debug']