try running ssh tests on ci

Author: minrk

.github/workflows/test.yml

@@ -18,13 +18,15 @@ jobs:
       fail-fast: false
       matrix:
         include:
+          - python: "3.9"
+            ssh: ssh
           - python: "3.6"
             tornado: "5.1.1"
           - python: "3.6"
           - python: "3.7"
             controller_ip: "*"
           - python: "3.8"
-            mpi: true
+            mpi: mpi
           - python: "3.9"
 
     steps:
@@ -42,6 +44,14 @@ jobs:
         run: |
           echo "IPP_CONTROLLER_IP=${{ matrix.controller_ip }}" >> $GITHUB_ENV
 
+      - name: Set up docker-compose for ssh launcher
+        if: ${{ matrix.ssh }}
+        run: |
+          export DOCKER_BUILDKIT=1
+          export COMPOSE_DOCKER_CLI_BUILD=1
+          cd ci/ssh
+          docker-compose up -d --build
+
       - name: Install Python (conda) ${{ matrix.python }}
         if: matrix.mpi
         run: |
@@ -76,8 +86,13 @@ jobs:
         run: |
           pytest -vx --color=yes --cov=ipyparallel ipyparallel/tests/test_cluster.py
 
+      - name: Run ssh tests
+        if: ${{ matrix.ssh }}
+        run: |
+          pytest -ra -v --maxfail=2 --color=yes --cov=ipyparallel ipyparallel/tests/test_ssh.py
+
       - name: Run tests
-        if: ${{ ! matrix.mpi }}
+        if: ${{ ! matrix.mpi && ! matrix.ssh }}
         # FIXME: --color=yes explicitly set because:
         #        https://github.com/actions/runner/issues/241
         run: |

ci/ssh/Dockerfile

@@ -31,7 +31,6 @@ ENV PIP_CACHE_DIR=/tmp/pip-cache
 COPY . /src/ipyparallel
 RUN --mount=type=cache,target=${PIP_CACHE_DIR} python3 -m pip install -e 'file:///src/ipyparallel#egg=ipyparallel[test]'
 
-EXPOSE 22
 # needed for sshd to start
 RUN mkdir /run/sshd
 # run sshd in the foreground

ci/ssh/docker-compose.yaml

@@ -5,4 +5,4 @@ services:
       context: ../..
       dockerfile: ci/ssh/Dockerfile
     ports:
-      - 2222:22
+      - "2222:22"

ipyparallel/tests/conftest.py

@@ -4,7 +4,7 @@
 import os
 import sys
 from subprocess import check_call
-from subprocess import STDOUT
+from subprocess import check_output
 from tempfile import TemporaryDirectory
 from unittest import mock
 
@@ -113,24 +113,37 @@ def ssh_dir(request):
     repo_root = os.path.abspath(os.path.join(ipp.__file__, os.pardir, os.pardir))
     ci_directory = os.environ.get("CI_DIR", os.path.join(repo_root, 'ci'))
     ssh_dir = os.path.join(ci_directory, "ssh")
-    # build image
-    check_call(["docker", "compose", "build"], cwd=ssh_dir)
-    # launch service
-    check_call(["docker", "compose", "up", "-d"], cwd=ssh_dir)
-    # shutdown service when we exit
-    request.addfinalizer(lambda: check_call(["docker", "compose", "down"], cwd=ssh_dir))
+
+    # only run ssh test if service was started before
+    try:
+        out = check_output(['docker-compose', 'ps', '-q'], cwd=ssh_dir)
+    except Exception:
+        pytest.skip("Needs docker compose")
+    else:
+        if not out.strip():
+            pytest.skip("ssh service not running")
+
+    # below is necessary for building/starting service as part of fixture
+    # currently we use whether the service is already started to decide whether to run the tests
+    # # build image
+    # check_call(["docker-compose", "build"], cwd=ssh_dir)
+    # # launch service
+    # check_call(["docker-compose", "up", "-d"], cwd=ssh_dir)
+    # # shutdown service when we exit
+    # request.addfinalizer(lambda: check_call(["docker-compose", "down"], cwd=ssh_dir))
     return ssh_dir
 
 
 @pytest.fixture
 def ssh_key(tmpdir, ssh_dir):
     key_file = tmpdir.join("id_rsa")
     check_call(
+        # this should be `docker compose cp sshd:...`
+        # but docker-compose 1.x doesn't support `cp` yet
         [
             'docker',
-            'compose',
             'cp',
-            'sshd:/home/ciuser/.ssh/id_rsa',
+            'ssh_sshd_1:/home/ciuser/.ssh/id_rsa',
             key_file,
         ],
         cwd=ssh_dir,